utils.py

import os
import re
import hashlib
import traceback
import hashlib
import json
import requests
import random
import subprocess
import logging
from collections import namedtuple
import datetime, calendar, pytz
from dateutil.tz import tzoffset

from .exceptions import ErrorMessage

# Setup logging
logger = logging.getLogger(__name__)


# Colormap (See https://bhaskarvk.github.io/colormap/reference/colormap.html)
color_map = ["#440154", "#440558", "#450a5c", "#450e60", "#451465", "#461969",
             "#461d6d", "#462372", "#472775", "#472c7a", "#46307c", "#45337d",
             "#433880", "#423c81", "#404184", "#3f4686", "#3d4a88", "#3c4f8a",
             "#3b518b", "#39558b", "#37598c", "#365c8c", "#34608c", "#33638d",
             "#31678d", "#2f6b8d", "#2d6e8e", "#2c718e", "#2b748e", "#29788e",
             "#287c8e", "#277f8e", "#25848d", "#24878d", "#238b8d", "#218f8d",
             "#21918d", "#22958b", "#23988a", "#239b89", "#249f87", "#25a186",
             "#25a584", "#26a883", "#27ab82", "#29ae80", "#2eb17d", "#35b479",
             "#3cb875", "#42bb72", "#49be6e", "#4ec16b", "#55c467", "#5cc863",
             "#61c960", "#6bcc5a", "#72ce55", "#7cd04f", "#85d349", "#8dd544",
             "#97d73e", "#9ed93a", "#a8db34", "#b0dd31", "#b8de30", "#c3df2e",
             "#cbe02d", "#d6e22b", "#e1e329", "#eae428", "#f5e626", "#fde725"]

#======================
#  Utility functions
#======================

def booleanize(*args, **kwargs):
    # Handle both single value and kwargs to get arg name
    name = None
    if args and not kwargs:
        value=args[0]
    elif kwargs and not args:
        for item in kwargs:
            name  = item
            value = kwargs[item]
            break
    else:
        raise Exception('Internal Error')
    
    # Handle shortcut: an arg with its name equal to ist value is considered as True
    if name==value:
        return True
    
    if isinstance(value, bool):
        return value
    else:
        if value.upper() in ('TRUE', 'YES', 'Y', '1'):
            return True
        else:
            return False


def send_email(to, subject, text):

    # Importing here instead of on top avoids circular dependencies problems when loading booleanize in settings
    from django.conf import settings
    
    if settings.DJANGO_EMAIL_SERVICE == 'Sendgrid':
        import sendgrid
        from sendgrid.helpers.mail import Email,Content,Mail

        sg = sendgrid.SendGridAPIClient(apikey=settings.DJANGO_EMAIL_APIKEY)
        from_email = Email(settings.DJANGO_EMAIL_FROM)
        to_email = Email(to)
        subject = subject
        content = Content('text/plain', text)
        mail = Mail(from_email, subject, to_email, content)
        
        try:
            response = sg.client.mail.send.post(request_body=mail.get())

            #logger.debug(response.status_code)
            #logger.debug(response.body)
            #logger.debug(response.headers)
        except Exception as e:
            logger.error(e)
        
        #logger.debug(response)
    

def format_exception(e, debug=False):
    
    # Importing here instead of on top avoids circular dependencies problems when loading booleanize in settings
    from django.conf import settings

    if settings.DEBUG:
        # Cutting away the last char removed the newline at the end of the stacktrace 
        return str('Got exception "{}" of type "{}" with traceback:\n{}'.format(e.__class__.__name__, type(e), traceback.format_exc()))[:-1]
    else:
        return str('Got exception "{}" of type "{}" with traceback "{}"'.format(e.__class__.__name__, type(e), traceback.format_exc().replace('\n', '|')))


def log_user_activity(level, msg, request, caller=None):

    # Get the caller function name through inspect with some logic
    #import inspect
    #caller =  inspect.stack()[1][3]
    #if caller == "post":
    #    caller =  inspect.stack()[2][3]
    
    try:
        msg = str(caller) + " view - USER " + str(request.user.email) + ": " + str(msg)
    except AttributeError:
        msg = str(caller) + " view - USER UNKNOWN: " + str(msg)

    try:
        level = getattr(logging, level)
    except:
        raise
    
    logger.log(level, msg)


def username_hash(email):
    '''Create md5 base 64 (25 chrars) hash from user email:'''             
    m = hashlib.md5()
    m.update(email)
    username = m.hexdigest().decode('hex').encode('base64')[:-3]
    return username


def random_username():
    '''Create a random string of 156 chars to be used as username'''             
    username = ''.join(random.choice('abcdefghilmnopqrtuvz') for _ in range(16))
    return username


def finalize_user_creation(user, auth='local'):

    from .models import Profile, KeyPair

    # Just an extra check
    try:
        Profile.objects.get(user=user)
    except Profile.DoesNotExist:
        pass
    else:
        raise Exception('Consistency error: already found a profile for user "{}"'.format(user))

    # Create profile
    logger.debug('Creating user profile for user "{}"'.format(user.email))
    Profile.objects.create(user=user, auth=auth)

    # Generate user keys
    out = os_shell('mkdir -p /data/resources/keys/', capture=True)
    if not out.exit_code == 0:
        logger.error(out)
        raise ErrorMessage('Something went wrong in creating user keys folder. Please contact support')
        
    command= "/bin/bash -c \"ssh-keygen -q -t rsa -N '' -C {}@rosetta -f /data/resources/keys/{}_id_rsa 2>/dev/null <<< y >/dev/null\"".format(user.email.split('@')[0], user.username)                        
    out = os_shell(command, capture=True)
    if not out.exit_code == 0:
        logger.error(out)
        raise ErrorMessage('Something went wrong in creating user keys. Please contact support')
        
    
    # Create key objects
    KeyPair.objects.create(user = user,
                          default = True,
                          private_key_file = '/data/resources/keys/{}_id_rsa'.format(user.username),
                          public_key_file = '/data/resources/keys/{}_id_rsa.pub'.format(user.username))
    

def sanitize_shell_encoding(text):
    return text.encode("utf-8", errors="ignore")


def format_shell_error(stdout, stderr, exit_code):
    
    string  = '\n#---------------------------------'
    string += '\n# Shell exited with exit code {}'.format(exit_code)
    string += '\n#---------------------------------\n'
    string += '\nStandard output: "'
    string += sanitize_shell_encoding(stdout)
    string += '"\n\nStandard error: "'
    string += sanitize_shell_encoding(stderr) +'"\n\n'
    string += '#---------------------------------\n'
    string += '# End Shell output\n'
    string += '#---------------------------------\n'

    return string


def os_shell(command, capture=False, verbose=False, interactive=False, silent=False):
    '''Execute a command in the OS shell. By default prints everything. If the capture switch is set,
    then it returns a namedtuple with stdout, stderr, and exit code.'''
    
    if capture and verbose:
        raise Exception('You cannot ask at the same time for capture and verbose, sorry')

    # Log command
    logger.debug('Shell executing command: "%s"', command)

    # Execute command in interactive mode    
    if verbose or interactive:
        exit_code = subprocess.call(command, shell=True)
        if exit_code == 0:
            return True
        else:
            return False

    # Execute command getting stdout and stderr
    # http://www.saltycrane.com/blog/2008/09/how-get-stdout-and-stderr-using-python-subprocess-module/
    
    process          = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True)
    (stdout, stderr) = process.communicate()
    exit_code        = process.wait()

    # Convert to str (Python 3)
    stdout = stdout.decode(encoding='UTF-8')
    stderr = stderr.decode(encoding='UTF-8')

    # Formatting..
    stdout = stdout[:-1] if (stdout and stdout[-1] == '\n') else stdout
    stderr = stderr[:-1] if (stderr and stderr[-1] == '\n') else stderr

    # Output namedtuple
    Output = namedtuple('Output', 'stdout stderr exit_code')

    if exit_code != 0:
        if capture:
            return Output(stdout, stderr, exit_code)
        else:
            print(format_shell_error(stdout, stderr, exit_code))      
            return False    
    else:
        if capture:
            return Output(stdout, stderr, exit_code)
        elif not silent:
            # Just print stdout and stderr cleanly
            print(stdout)
            print(stderr)
            return True
        else:
            return True


def get_md5(string):
    if not string:
        raise Exception("Colund not compute md5 of empty/None value")
    
    m = hashlib.md5()
    
    # Fix for Python3
    try:
        if isinstance(string,unicode):
            string=string.encode('utf-8')
    except NameError:
        string=string.encode('utf-8')
        
    m.update(string)
    md5 = str(m.hexdigest())
    return md5


#=========================
#   Time 
#=========================

def timezonize(timezone):
    '''Convert a string representation of a timezone to its pytz object or do nothing if the argument is already a pytz timezone'''
    
    # Check if timezone is a valid pytz object is hard as it seems that they are spread arount the pytz package.
    # Option 1): Try to convert if string or unicode, else try to
    # instantiate a datetiem object with the timezone to see if it is valid 
    # Option 2): Get all memebers of the pytz package and check for type, see
    # http://stackoverflow.com/questions/14570802/python-check-if-object-is-instance-of-any-class-from-a-certain-module
    # Option 3) perform a hand.made test. We go for this one, tests would fail if it gets broken
    
    if not 'pytz' in str(type(timezone)):
        timezone = pytz.timezone(timezone)
    return timezone

def now_t():
    '''Return the current time in epoch seconds'''
    return now_s()

def now_s():
    '''Return the current time in epoch seconds'''
    return calendar.timegm(now_dt().utctimetuple())

def now_dt(tzinfo='UTC'):
    '''Return the current time in datetime format'''
    if tzinfo != 'UTC':
        raise NotImplementedError()
    return datetime.datetime.utcnow().replace(tzinfo = pytz.utc)

def dt(*args, **kwargs):
    '''Initialize a datetime object in the proper way. Using the standard datetime leads to a lot of
     problems with the tz package. Also, it forces UTC timezone if no timezone is specified'''
    
    if 'tz' in kwargs:
        tzinfo = kwargs.pop('tz')
    else:
        tzinfo  = kwargs.pop('tzinfo', None)
        
    offset_s  = kwargs.pop('offset_s', None)   
    trustme   = kwargs.pop('trustme', None)
    
    if kwargs:
        raise Exception('Unhandled arg: "{}".'.format(kwargs))
        
    if (tzinfo is None):
        # Force UTC if None
        timezone = timezonize('UTC')
        
    else:
        timezone = timezonize(tzinfo)
    
    if offset_s:
        # Special case for the offset
        if not tzoffset:
            raise Exception('For ISO date with offset please install dateutil')
        time_dt = datetime.datetime(*args, tzinfo=tzoffset(None, offset_s))
    else:
        # Standard  timezone
        time_dt = timezone.localize(datetime.datetime(*args))

    # Check consistency    
    if not trustme and timezone != pytz.UTC:
        if not check_dt_consistency(time_dt):
            raise Exception('Sorry, time {} does not exists on timezone {}'.format(time_dt, timezone))

    return  time_dt

def get_tz_offset_s(time_dt):
    '''Get the time zone offset in seconds'''
    return s_from_dt(time_dt.replace(tzinfo=pytz.UTC)) - s_from_dt(time_dt)

def check_dt_consistency(date_dt):
    '''Check that the timezone is consistent with the datetime (some conditions in Python lead to have summertime set in winter)'''

    # https://en.wikipedia.org/wiki/Tz_database
    # https://www.iana.org/time-zones
    
    if date_dt.tzinfo is None:
        return True
    else:
        
        # This check is quite heavy but there is apparently no other way to do it.
        if date_dt.utcoffset() != dt_from_s(s_from_dt(date_dt), tz=date_dt.tzinfo).utcoffset():
            return False
        else:
            return True

def correct_dt_dst(datetime_obj):
    '''Check that the dst is correct and if not change it'''

    # https://en.wikipedia.org/wiki/Tz_database
    # https://www.iana.org/time-zones

    if datetime_obj.tzinfo is None:
        return datetime_obj

    # Create and return a New datetime object. This corrects the DST if errors are present.
    return dt(datetime_obj.year,
              datetime_obj.month,
              datetime_obj.day,
              datetime_obj.hour,
              datetime_obj.minute,
              datetime_obj.second,
              datetime_obj.microsecond,
              tzinfo=datetime_obj.tzinfo)

def change_tz(dt, tz):
    return dt.astimezone(timezonize(tz))

def dt_from_t(timestamp_s, tz=None):
    '''Create a datetime object from an epoch timestamp in seconds. If no timezone is given, UTC is assumed'''
    # TODO: check if uniform everything on this one or not.
    return dt_from_s(timestamp_s=timestamp_s, tz=tz)
    
def dt_from_s(timestamp_s, tz=None):
    '''Create a datetime object from an epoch timestamp in seconds. If no timezone is given, UTC is assumed'''

    if not tz:
        tz = "UTC"
    try:
        timestamp_dt = datetime.datetime.utcfromtimestamp(float(timestamp_s))
    except TypeError:
        raise Exception('timestamp_s argument must be string or number, got {}'.format(type(timestamp_s)))

    pytz_tz = timezonize(tz)
    timestamp_dt = timestamp_dt.replace(tzinfo=pytz.utc).astimezone(pytz_tz)
    
    return timestamp_dt

def s_from_dt(dt):
    '''Returns seconds with floating point for milliseconds/microseconds.'''
    if not (isinstance(dt, datetime.datetime)):
        raise Exception('s_from_dt function called without datetime argument, got type "{}" instead.'.format(dt.__class__.__name__))
    microseconds_part = (dt.microsecond/1000000.0) if dt.microsecond else 0
    return  ( calendar.timegm(dt.utctimetuple()) + microseconds_part)

def dt_from_str(string, timezone=None):

    # Supported formats on UTC
    # 1) YYYY-MM-DDThh:mm:ssZ
    # 2) YYYY-MM-DDThh:mm:ss.{u}Z

    # Supported formats with offset    
    # 3) YYYY-MM-DDThh:mm:ss+ZZ:ZZ
    # 4) YYYY-MM-DDThh:mm:ss.{u}+ZZ:ZZ

    # Split and parse standard part
    date, time = string.split('T')
    
    if time.endswith('Z'):
        # UTC
        offset_s = 0
        time = time[:-1]
        
    elif ('+') in time:
        # Positive offset
        time, offset = time.split('+')
        # Set time and extract positive offset
        offset_s = (int(offset.split(':')[0])*60 + int(offset.split(':')[1]) )* 60   
        
    elif ('-') in time:
        # Negative offset
        time, offset = time.split('-')
        # Set time and extract negative offset
        offset_s = -1 * (int(offset.split(':')[0])*60 + int(offset.split(':')[1])) * 60      
    
    else:
        raise Exception('Format error')
    
    # Handle time
    hour, minute, second = time.split(':')
    
    # Now parse date (easy)
    year, month, day = date.split('-') 

    # Convert everything to int
    year    = int(year)
    month   = int(month)
    day     = int(day)
    hour    = int(hour)
    minute  = int(minute)
    if '.' in second:
        usecond = int(second.split('.')[1])
        second  = int(second.split('.')[0])
    else:
        second  = int(second)
        usecond = 0
    
    return dt(year, month, day, hour, minute, second, usecond, offset_s=offset_s)

def dt_to_str(dt):
    '''Return the ISO representation of the datetime as argument'''
    return dt.isoformat()

class dt_range(object):

    def __init__(self, from_dt, to_dt, timeSlotSpan):

        self.from_dt      = from_dt
        self.to_dt        = to_dt
        self.timeSlotSpan = timeSlotSpan

    def __iter__(self):
        self.current_dt = self.from_dt
        return self

    def __next__(self):

        # Iterator logic
        if self.current_dt > self.to_dt:
            raise StopIteration
        else:
            prev_current_dt = self.current_dt
            self.current_dt = self.current_dt + self.timeSlotSpan
            return prev_current_dt

    # Python 2.x
    def next(self):
        return self.__next__()


#================================
#  Others
#================================

def debug_param(**kwargs):
    for item in kwargs:
        logger.critical('Param "{}": "{}"'.format(item, kwargs[item]))

def get_my_ip():
    import socket
    hostname = socket.gethostname()
    my_ip = socket.gethostbyname(hostname)
    return my_ip

def get_webapp_conn_string():
    webapp_host = os.environ.get('ROSETTA_WEBAPP_HOST', get_my_ip())
    webapp_port = os.environ.get('ROSETTA_WEBAPP_PORT', '8080')
    webapp_conn_string = 'http://{}:{}'.format(webapp_host, webapp_port)
    return webapp_conn_string

def get_platform_registry():
    platform_registry_host = os.environ.get('PLATFORM_REGISTRY_HOST', 'proxy')
    platform_registry_port = os.environ.get('PLATFORM_REGISTRY_PORT', '5000')
    platform_registry_conn_string = '{}:{}'.format(platform_registry_host, platform_registry_port)
    return platform_registry_conn_string
    
def get_rosetta_tasks_tunnel_host():
    # Importing here instead of on top avoids circular dependencies problems when loading booleanize in settings
    from django.conf import settings
    tunnel_host = os.environ.get('ROSETTA_TASKS_TUNNEL_HOST', settings.ROSETTA_HOST)
    return tunnel_host

def get_rosetta_tasks_proxy_host():
    # Importing here instead of on top avoids circular dependencies problems when loading booleanize in settings
    from django.conf import settings
    proxy_host = os.environ.get('ROSETTA_TASKS_PROXY_HOST', settings.ROSETTA_HOST)
    return proxy_host

def hash_string_to_int(string):
    return int(hashlib.sha1(string.encode('utf8')).hexdigest(), 16)



#================================
#  Tunnel (and proxy) setup
#================================

def setup_tunnel_and_proxy(task):

    # Importing here instead of on top avoids circular dependencies problems when loading booleanize in settings
    from .models import Task, KeyPair, TaskStatuses
    
    # If there is no tunnel port allocated yet, find one
    if not task.tcp_tunnel_port:

        # Get a free port fot the tunnel:
        allocated_tcp_tunnel_ports = []
        for other_task in Task.objects.all():
            if other_task.tcp_tunnel_port and not other_task.status in [TaskStatuses.exited, TaskStatuses.stopped]:
                allocated_tcp_tunnel_ports.append(other_task.tcp_tunnel_port)

        if task.requires_proxy:
            # Proxy ports are in the 9000-range
            for port in range(9000, 9021):
                if not port in allocated_tcp_tunnel_ports:
                    tcp_tunnel_port = port
                    break
        else:
            # Direct tunnel ports are in the 7000-range
            for port in range(7000, 7021):
                if not port in allocated_tcp_tunnel_ports:
                    tcp_tunnel_port = port
                    break

        if not tcp_tunnel_port:
            logger.error('Cannot find a free port for the tunnel for task "{}"'.format(task))
            raise ErrorMessage('Cannot find a free port for the tunnel to the task')

        task.tcp_tunnel_port = tcp_tunnel_port
        task.save()


    # Check if the tunnel is (still) active, if not create it
    logger.debug('Checking if task "{}" has a running tunnel'.format(task))

    out = os_shell('ps -ef | grep ":{}:{}:{}" | grep -v grep'.format(task.tcp_tunnel_port, task.interface_ip, task.interface_port), capture=True)

    if out.exit_code == 0:
        logger.debug('Task "{}" has a running tunnel, using it'.format(task))
    else:
        logger.debug('Task "{}" has no running tunnel, creating it'.format(task))

        # Get user keys
        user_keys = KeyPair.objects.get(user=task.user, default=True)

        # Tunnel command
        if task.computing.type == 'remotehop':           
            
            # Get computing params
            first_host = task.computing.conf.get('first_host')
            first_user = task.computing.conf.get('first_user')
            #second_host = task.computing.conf.get('second_host')
            #second_user = task.computing.conf.get('second_user')
            #setup_command = task.computing.conf.get('setup_command')
            #base_port = task.computing.conf.get('base_port')
                     
            tunnel_command= 'ssh -4 -i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(user_keys.private_key_file, task.tcp_tunnel_port, task.interface_ip, task.interface_port, first_user, first_host)

        else:
            tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tcp_tunnel_port, task.interface_ip, task.interface_port)
        
        background_tunnel_command = 'nohup {} >/dev/null 2>&1 &'.format(tunnel_command)

        # Log
        logger.debug('Opening tunnel with command: {}'.format(background_tunnel_command))

        # Execute
        subprocess.Popen(background_tunnel_command, shell=True)

  
    # Setup the proxy now (if required.)
    if task.requires_proxy:
        
        # Ensure conf directory exists
        if not os.path.exists('/shared/etc_apache2_sites_enabled'):
            os.makedirs('/shared/etc_apache2_sites_enabled')
    
        # Set conf file name
        apache_conf_file = '/shared/etc_apache2_sites_enabled/{}.conf'.format(task.uuid)
    
        # Check if proxy conf exists 
        if not os.path.exists(apache_conf_file):
    
            # Write conf file
            # Some info about the various SSL switches: https://serverfault.com/questions/577616/using-https-between-apache-loadbalancer-and-backends
            logger.debug('Writing task proxy conf to {}'.format(apache_conf_file))
            websocket_protocol = 'wss' if task.container.interface_protocol == 'https' else 'ws'
            rosetta_tasks_proxy_host = get_rosetta_tasks_proxy_host()
            apache_conf_content = '''
#---------------------------
#  Task interface proxy 
#---------------------------

Listen '''+str(task.tcp_tunnel_port)+'''

<VirtualHost *:'''+str(task.tcp_tunnel_port)+'''>
    ServerAdmin admin@rosetta.platform
    SSLEngine on
    SSLCertificateFile /root/certificates/rosetta_platform/rosetta_platform.crt
    SSLCertificateKeyFile /root/certificates/rosetta_platform/rosetta_platform.key
    SSLCACertificateFile /root/certificates/rosetta_platform/rosetta_platform.ca-bundle
    DocumentRoot /var/www/html
</VirtualHost>

<VirtualHost *:'''+str(task.tcp_tunnel_port)+'''>
    
    ServerName  '''+rosetta_tasks_proxy_host+'''
    ServerAdmin admin@rosetta.platform
    
    SSLEngine on
    SSLCertificateFile /root/certificates/rosetta_platform/rosetta_tasks.crt
    SSLCertificateKeyFile /root/certificates/rosetta_platform/rosetta_tasks.key
    SSLCACertificateFile /root/certificates/rosetta_platform/rosetta_tasks.ca-bundle
    
    SSLProxyEngine On
    SSLProxyVerify none 
    SSLProxyCheckPeerCN off
    SSLProxyCheckPeerName off  

    BrowserMatch "MSIE [2-6]" \
        nokeepalive ssl-unclean-shutdown \
        downgrade-1.0 force-response-1.0
    BrowserMatch "MSIE [17-9]" ssl-unclean-shutdown

    # Use RewriteEngine to handle websocket connection upgrades
    RewriteEngine On
    RewriteCond %{HTTP:Connection} Upgrade [NC]
    RewriteCond %{HTTP:Upgrade} websocket [NC]
    RewriteRule /(.*) '''+str(websocket_protocol)+'''://webapp:'''+str(task.tcp_tunnel_port)+'''/$1 [P,L]

    <Location "/">
      AuthType Basic
      AuthName "Restricted area"
      AuthUserFile /shared/etc_apache2_sites_enabled/'''+str(task.uuid)+'''.htpasswd
      Require valid-user  

      # preserve Host header to avoid cross-origin problems
      ProxyPreserveHost on
      # proxy to the port
      ProxyPass         '''+str(task.container.interface_protocol)+'''://webapp:'''+str(task.tcp_tunnel_port)+'''/
      ProxyPassReverse  '''+str(task.container.interface_protocol)+'''://webapp:'''+str(task.tcp_tunnel_port)+'''/
    </Location>

</VirtualHost>

'''
            with open(apache_conf_file, 'w') as f:
                f.write(apache_conf_content)
    
        # Now check if conf exist on proxy
        logger.debug('Checking if conf is enabled on proxy service')
        out = os_shell('ssh -o StrictHostKeyChecking=no proxy "[ -e /etc/apache2/sites-enabled/{}.conf ]"'.format(task.uuid), capture=True)
    
        if out.exit_code == 1:
      
            logger.debug('Conf not enabled on proxy service, linkig it and reloading Apache conf')
      
            # Link on proxy since conf does not exist
            out = os_shell('ssh -o StrictHostKeyChecking=no proxy "sudo ln -s /shared/etc_apache2_sites_enabled/{0}.conf /etc/apache2/sites-enabled/{0}.conf"'.format(task.uuid), capture=True)
            if out.exit_code != 0:
                logger.error(out.stderr)
                raise ErrorMessage('Somthing went wrong when activating the task proxy conf')        
            
            # Reload apache conf on Proxy
            out = os_shell('ssh -o StrictHostKeyChecking=no proxy "sudo apache2ctl graceful"', capture=True)
            if out.exit_code != 0:
                logger.error(out.stderr) 
                raise ErrorMessage('Somthing went wrong when loading the task proxy conf')        
            



def get_ssh_access_mode_credentials(computing, user):
    from .models import KeyPair
    # Get computing host
    try:
        computing_host = computing.conf.get('host')
    except AttributeError:
        computing_host = None
    if not computing_host:
        raise ValueError('No computing host?!')
            
    # Get computing user and keys
    if computing.auth_mode == 'user_keys':
        computing_user = user.profile.get_extra_conf('computing_user', computing)
        if not computing_user:
            raise ValueError('No \'computing_user\' parameter found for computing resource \'{}\' in user profile'.format(computing.name))
        # Get user key
        computing_keys = KeyPair.objects.get(user=user, default=True)
    elif computing.auth_mode == 'platform_keys':        
        computing_user = computing.conf.get('user')
        computing_keys = KeyPair.objects.get(user=None, default=True)
    else:
        raise NotImplementedError('Auth modes other than user_keys and platform_keys not supported.')
    if not computing_user:
            raise ValueError('No \'user\' parameter found for computing resource \'{}\' in its configuration'.format(computing.name))
    return (computing_user, computing_host, computing_keys)



def sanitize_container_env_vars(env_vars):
    
    for env_var in env_vars:
        
        # Check only alphanumeric chars, slashed, dashes and underscores
        if not re.match("^[/A-Za-z0-9_-]*$", env_vars[env_var]):
            raise ValueError('Value "{}" for env var "{}" is not valid: only alphanumeric, slashes, dashes and underscores are.'.format(env_vars[env_var], env_var))

    return env_vars


def get_or_create_container_from_repository(user, repository_url, repository_tag=None, container_name=None, container_description=None): 
    
    from .models import Container    
    logger.debug('Called get_or_create_container_from_repository with repository_url="{}" and repository_tag="{}"'.format(repository_url,repository_tag))

    # Set repo name
    repository_name = '{}/{}'.format(repository_url.split('/')[-2],repository_url.split('/')[-1])

    # If building:
    #{"message": "Successfully built 5a2089b2c334\n", "phase": "building"}
    #{"message": "Successfully tagged r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3:latest\n", "phase": "building"}
    
    # If reusing:
    #{"message": "Reusing existing image (r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3), not building."}
    
    appendix = 'CMD ["jupyter", "notebook", "--ip", "0.0.0.0", "--NotebookApp.token", ""]'

    
    # Build the Docker container for this repo
    if repository_tag:
        command = 'sudo jupyter-repo2docker --ref {} --user-id 1000 --user-name jovyan --no-run --appendix \'{}\' --json-logs {}'.format(repository_tag, appendix, repository_url)
    else:
        command = 'sudo jupyter-repo2docker --user-id 1000 --user-name jovyan --no-run --appendix \'{}\' --json-logs {}'.format(repository_url, appendix)
    out = os_shell(command, capture=True)
    if out.exit_code != 0:
        logger.error(out.stderr)
        raise ErrorMessage('Something went wrong when creating the Dockerfile for repository "{}"'.format(repository_url))   

    # Convert output to lines
    out_lines = out.stderr.split('\n')

    # Get rep2docker image name from output. Use "strip()" as sometimes the newline chars might jump in.
    last_line_message = json.loads(out_lines[-1])['message']
    if 'Reusing existing image' in last_line_message:
        repo2docker_image_name = last_line_message.split('(')[1].split(')')[0].strip()
    elif 'Successfully tagged' in last_line_message:
        repo2docker_image_name = last_line_message.split(' ')[2].strip()
    else:
        raise Exception('Cannot build')

    # Set image registry, name and tag. Use "strip()" as sometimes the newline chars might jump in.
    registry = get_platform_registry()
    image_name = repository_name.lower().strip()
    if repo2docker_image_name.endswith(':latest'):
        # Not clear why sometimes this happens. maybe if an existent image gets reused?
        image_name_for_tag = repo2docker_image_name.replace(':latest','')
    else:
        image_name_for_tag = repo2docker_image_name
    
    image_tag = image_name_for_tag[-7:] # The last part of the image name generated by repo2docker is the git short hash

    # Re-tag image taking into account that if we are using the proxy as registry we use localhost or it won't work
    if registry == 'proxy:5000':
        push_registry = 'localhost:5000'
    else:
        push_registry = registry
        
    out = os_shell('sudo docker tag {} {}/{}:{}'.format(repo2docker_image_name,push_registry,image_name,image_tag) , capture=True)
    if out.exit_code != 0:
        logger.error(out.stderr)
        raise ErrorMessage('Something went wrong when tagging the container for repository "{}"'.format(repository_url))   

    # Push image to the (local) registry
    out = os_shell('sudo docker push {}/{}:{}'.format(push_registry,image_name,image_tag) , capture=True)
    if out.exit_code != 0:
        logger.error(out.stderr)
        raise ErrorMessage('Something went wrong when pushing the container for repository "{}"'.format(repository_url))   

    # Create the container if not already existent
    try:
        container = Container.objects.get(user=user, registry=registry, image_name=image_name, image_tag=image_tag)
    except Container.DoesNotExist:

        # Get name repo name and description from remote if we have and if we can
        repository_name_from_source= None
        repository_description_from_source = None
        if not container_name or not container_description:
            if repository_url.startswith('https://github.com'):
                try:
                    response = requests.get('https://api.github.com/repos/{}'.format(repository_name))
                    json_content = json.loads(response.content)
                    repository_name_from_source = json_content['name'].title()
                    repository_description_from_source = json_content['description']
                    if not repository_description_from_source.endswith('.'):
                        repository_description_from_source+='.'
                    repository_description_from_source += ' Built from {}'.format(repository_url)
                except:
                    pass
        
        # Set default container name and description            
        if not container_name:
            container_name = repository_name_from_source if repository_name_from_source else repository_name
        
        if not container_description:
            container_description = repository_description_from_source if repository_description_from_source else 'Built from {}'.format(repository_url)

        # Ok, create the container
        container = Container.objects.create(user = user,
                                             name = container_name,
                                             description = container_description,
                                             registry = registry,
                                             image_name = image_name,
                                             image_tag  = image_tag,
                                             image_arch = 'amd64',
                                             image_os = 'linux',
                                             interface_port = '8888',
                                             interface_protocol = 'http',
                                             interface_transport = 'tcp/ip',
                                             supports_custom_interface_port = False,
                                             supports_interface_auth = False)
    return container