Commit 3b716a0e authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Added support for custom SSH ports in computing resources using SSH access mode.

parent 36f4f515

services/webapp/code/rosetta/core_app/api.py

+5 −5
Original line number Original line Diff line number Diff line
@@ -405,13 +405,13 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): 
            dest = dest.replace('\ ', '\\\\\\ ')

            dest = dest.replace('\ ', '\\\\\\ ')

        

        

        # Get credentials

        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(computing, user)

        computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(computing, user)





        # Command

        # Command

        if mode=='get':

        if mode=='get':

            command = 'scp -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{}:{} {}'.format(computing_keys.private_key_file, computing_user, computing_host, source, dest)

            command = 'scp -P {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{}:{} {}'.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, source, dest)

        elif mode == 'put':

        elif mode == 'put':

            command = 'scp -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {} {}@{}:{}'.format(computing_keys.private_key_file, source, computing_user, computing_host, dest)

            command = 'scp -P -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {} {}@{}:{}'.format(computing_port, computing_keys.private_key_file, source, computing_user, computing_host, dest)

        else:

        else:

            raise ValueError('Unknown mode "{}"'.format(mode))

            raise ValueError('Unknown mode "{}"'.format(mode))
@@ -423,10 +423,10 @@ class FileManagerAPI(PrivateGETAPI, PrivatePOSTAPI): 
        if storage.access_mode == 'ssh+cli':

        if storage.access_mode == 'ssh+cli':

            if storage.access_through_computing:

            if storage.access_through_computing:

                # Get credentials

                # Get credentials

                computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(storage.computing, user)

                computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(storage.computing, user)

        

        

                # Command

                # Command

                command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} "{}"'.format(computing_keys.private_key_file, computing_user, computing_host, command)

                command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} "{}"'.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, command)

            else:

            else:

                raise NotImplementedError('Not accessing through computing is not implemented for storage type "{}"'.format(storage.type))               

                raise NotImplementedError('Not accessing through computing is not implemented for storage type "{}"'.format(storage.type))               

        elif storage.access_mode == 'cli':

        elif storage.access_mode == 'cli':

services/webapp/code/rosetta/core_app/computing_managers.py

+12 −12
Original line number Original line Diff line number Diff line
@@ -203,7 +203,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
        logger.debug('Starting a remote task "{}"'.format(self.computing))

        logger.debug('Starting a remote task "{}"'.format(self.computing))





        # Get credentials

        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)





        # Get webapp conn string

        # Get webapp conn string

        from.utils import get_webapp_conn_string

        from.utils import get_webapp_conn_string
@@ -270,7 +270,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
                    else:

                    else:

                        binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)

                        binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)

            

            

            run_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)

            run_command  = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host)

            run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid) 

            run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid) 

            run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT {} {} &&'.format(authstring, varsstring)

            run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT {} {} &&'.format(authstring, varsstring)
@@ -333,7 +333,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
            # TODO: remove this hardcoding

            # TODO: remove this hardcoding

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''

            prefix = 'sudo' if (computing_host == 'slurmclusterworker' and container_engine=='docker') else ''

            

            

            run_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)

            run_command  = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host)

            run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid) 

            run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir /tmp/{}_data && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid) 

            run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'wget {} {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export TASK_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds)        

            run_command += 'exec nohup {} {} run -p \$TASK_PORT:{} {} {} {} '.format(prefix, container_engine, task.container.interface_port, authstring, varsstring, binds)
@@ -367,7 +367,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
    def _stop_task(self, task, **kwargs):

    def _stop_task(self, task, **kwargs):





        # Get credentials

        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)





        # Handle container engine

        # Handle container engine

        container_engine = None

        container_engine = None
@@ -385,7 +385,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
        else:

        else:

            raise NotImplementedError('Container engine {} not supported'.format(container_engine))

            raise NotImplementedError('Container engine {} not supported'.format(container_engine))





        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_keys.private_key_file, computing_user, computing_host, internal_stop_command)

        stop_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, internal_stop_command)

        out = os_shell(stop_command, capture=True)

        out = os_shell(stop_command, capture=True)

        if out.exit_code != 0:

        if out.exit_code != 0:

            if ('No such process' in out.stderr) or ('No such container' in out.stderr) or ('no container' in out.stderr) or ('missing' in out.stderr):

            if ('No such process' in out.stderr) or ('No such container' in out.stderr) or ('no container' in out.stderr) or ('missing' in out.stderr):
@@ -401,7 +401,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
    def _get_task_log(self, task, **kwargs):

    def _get_task_log(self, task, **kwargs):





        # Get credentials

        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        

        

        # Handle container engine

        # Handle container engine

        container_engine = None

        container_engine = None
@@ -421,7 +421,7 @@ class SSHStandaloneComputingManager(StandaloneComputingManager, SSHComputingMana 
            raise NotImplementedError('Container engine {} not supported'.format(container_engine))

            raise NotImplementedError('Container engine {} not supported'.format(container_engine))

            

            

        # Prepare full comand

        # Prepare full comand

        view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_keys.private_key_file, computing_user, computing_host, internal_view_log_command)

        view_log_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "{}"\''.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, internal_view_log_command)





        # Execute

        # Execute

        out = os_shell(view_log_command, capture=True)

        out = os_shell(view_log_command, capture=True)
@@ -437,7 +437,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag 
        logger.debug('Starting a remote task "{}"'.format(self.computing))

        logger.debug('Starting a remote task "{}"'.format(self.computing))





        # Get credentials

        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)





        # Get webapp conn string

        # Get webapp conn string

        from.utils import get_webapp_conn_string

        from.utils import get_webapp_conn_string
@@ -524,7 +524,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag 
                    else:

                    else:

                        binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)

                        binds += ',{}:{}'.format(expanded_base_path, expanded_bind_path)





            run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_keys.private_key_file, computing_user, computing_host)

            run_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(computing_port, computing_keys.private_key_file, computing_user, computing_host)

            run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {} {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {} {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(CHECK_WGET_CERT_STR, webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT {} {} && '.format(authstring, varsstring)

            run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT {} {} && '.format(authstring, varsstring)

            run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
@@ -568,7 +568,7 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag 
    def _stop_task(self, task, **kwargs):

    def _stop_task(self, task, **kwargs):

        

        

        # Get credentials

        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)





        # Stop the task remotely

        # Stop the task remotely

        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(computing_keys.private_key_file, computing_user, computing_host, task.id)

        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(computing_keys.private_key_file, computing_user, computing_host, task.id)
@@ -584,10 +584,10 @@ class SlurmSSHClusterComputingManager(ClusterComputingManager, SSHComputingManag 
    def _get_task_log(self, task, **kwargs):

    def _get_task_log(self, task, **kwargs):

        

        

        # Get credentials

        # Get credentials

        computing_user, computing_host, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)

        computing_user, computing_host, computing_port, computing_keys = get_ssh_access_mode_credentials(self.computing, task.user)





        # View log remotely

        # View log remotely

        view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(computing_keys.private_key_file, computing_user, computing_host, task.uuid)

        view_log_command = 'ssh -p {} -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(computing_port, computing_keys.private_key_file, computing_user, computing_host, task.uuid)





        out = os_shell(view_log_command, capture=True)

        out = os_shell(view_log_command, capture=True)

        if out.exit_code != 0:

        if out.exit_code != 0:

services/webapp/code/rosetta/core_app/utils.py

+12 −2
Original line number Original line Diff line number Diff line
@@ -716,7 +716,9 @@ Listen '''+str(task.tcp_tunnel_port)+''' 








def get_ssh_access_mode_credentials(computing, user):

def get_ssh_access_mode_credentials(computing, user):

    

    from .models import KeyPair

    from .models import KeyPair

    

    # Get computing host

    # Get computing host

    try:

    try:

        computing_host = computing.conf.get('host')

        computing_host = computing.conf.get('host')
@@ -725,6 +727,14 @@ def get_ssh_access_mode_credentials(computing, user): 
    if not computing_host:

    if not computing_host:

        raise ValueError('No computing host?!')

        raise ValueError('No computing host?!')





    # Get computing (SSH) port

    try:

        computing_port = computing.conf.get('port')

    except AttributeError:

        computing_port = 22

    if not computing_host:

        computing_port = 22

      

    # Get computing user and keys

    # Get computing user and keys

    if computing.auth_mode == 'user_keys':

    if computing.auth_mode == 'user_keys':

        computing_user = user.profile.get_extra_conf('computing_user', computing)

        computing_user = user.profile.get_extra_conf('computing_user', computing)
@@ -739,7 +749,7 @@ def get_ssh_access_mode_credentials(computing, user): 
        raise NotImplementedError('Auth modes other than user_keys and platform_keys not supported.')

        raise NotImplementedError('Auth modes other than user_keys and platform_keys not supported.')

    if not computing_user:

    if not computing_user:

            raise ValueError('No \'user\' parameter found for computing resource \'{}\' in its configuration'.format(computing.name))

            raise ValueError('No \'user\' parameter found for computing resource \'{}\' in its configuration'.format(computing.name))

    return (computing_user, computing_host, computing_keys)

    return (computing_user, computing_host, computing_port, computing_keys)