Commit 8fb5e8df authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Added support for uWSGI together with static files serivng. Added the... 

Added support for uWSGI together with static files serivng. Added the DJANGO_DEV_SERVER env var to switch between the dev server and uWSGI. Small changes in error handling to play nicer with logs when not running in dev mode.
parent e43feab1

services/webapp/code/rosetta/core_app/api.py

+54 −74
Original line number Diff line number Diff line
@@ -209,7 +209,6 @@ class UserViewSet(viewsets.ModelViewSet): 
class agent_api(PublicGETAPI):

    

    def _get(self, request):

        try:

        

        task_uuid = request.GET.get('task_uuid', None)

        if not task_uuid:
@@ -326,23 +325,4 @@ print(port) 
            return HttpResponse('Unknown action "{}"'.format(action))





        except Exception as e:

            logger.error(e)

services/webapp/code/rosetta/core_app/computing_managers.py

+9 −9
Original line number Diff line number Diff line
@@ -205,7 +205,7 @@ class RemoteComputingManager(ComputingManager): 
                else:

                    binds += ',{}'.format(task.extra_binds)

            

            run_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)

            run_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)

            run_command += '/bin/bash -c \'"rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp && mkdir -p /tmp/{}_data/home && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid) 

            run_command += 'wget {}/api/v1/base/agent/?task_uuid={} -O /tmp/{}_data/agent.py &> /dev/null && export BASE_PORT=\$(python /tmp/{}_data/agent.py 2> /tmp/{}_data/task.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
@@ -262,7 +262,7 @@ class RemoteComputingManager(ComputingManager): 
        user = task.computing.get_conf_param('user')



        # Stop the task remotely

        stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)

        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "kill -9 {}"\''.format(user_keys.private_key_file, user, host, task.pid)

        out = os_shell(stop_command, capture=True)

        if out.exit_code != 0:

            if not 'No such process' in out.stderr:
@@ -286,7 +286,7 @@ class RemoteComputingManager(ComputingManager): 
        user = task.computing.get_conf_param('user')



        # View log remotely

        view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)

        view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat /tmp/{}_data/task.log"\''.format(user_keys.private_key_file, user, host, task.uuid)



        out = os_shell(view_log_command, capture=True)

        if out.exit_code != 0:
@@ -364,7 +364,7 @@ class SlurmComputingManager(ComputingManager): 
                else:

                    binds += ',{}'.format(task.extra_binds)



            run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)

            run_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)

            run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)

            run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)

            run_command += 'rm -rf /tmp/{}_data && mkdir -p /tmp/{}_data/tmp &>> \$HOME/{}.log && mkdir -p /tmp/{}_data/home &>> \$HOME/{}.log && chmod 700 /tmp/{}_data && '.format(task.uuid, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
@@ -431,7 +431,7 @@ class SlurmComputingManager(ComputingManager): 
        user = task.computing.get_conf_param('user')



        # Stop the task remotely

        stop_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)

        stop_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "scancel {}"\''.format(user_keys.private_key_file, user, host, task.pid)

        out = os_shell(stop_command, capture=True)

        if out.exit_code != 0:

            raise Exception(out.stderr)
@@ -454,7 +454,7 @@ class SlurmComputingManager(ComputingManager): 
        user = task.computing.get_conf_param('user')



        # View log remotely

        view_log_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)

        view_log_command = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} \'/bin/bash -c "cat \$HOME/{}.log"\''.format(user_keys.private_key_file, user, host, task.uuid)



        out = os_shell(view_log_command, capture=True)

        if out.exit_code != 0:
@@ -518,7 +518,7 @@ class RemotehopComputingManager(ComputingManager): 
                else:

                    binds += ',{}'.format(task.extra_binds)



            run_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)

            run_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)

            run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)

            

            if use_agent:
@@ -592,7 +592,7 @@ class RemotehopComputingManager(ComputingManager): 
        second_user = task.computing.get_conf_param('second_user')



        # Stop the task remotely

        stop_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)

        stop_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)

        stop_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)

        stop_command += 'kill -9 {}"'.format(task.pid)
@@ -621,7 +621,7 @@ class RemotehopComputingManager(ComputingManager): 
        second_user = task.computing.get_conf_param('second_user')



        # View log remotely

        view_log_command  = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)

        view_log_command  = 'ssh -o LogLevel=ERROR -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)

        view_log_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} '.format(second_user, second_host)

        view_log_command += 'cat \\\\\\$HOME/{}.log"'.format(task.uuid)

services/webapp/code/rosetta/core_app/decorators.py

+13 −17
Original line number Diff line number Diff line
@@ -51,18 +51,16 @@ def public_view(wrapped_view): 
                error_text = str(e)

            else:



                # Raise te exception if we are in debug mode

                # Log the exception 

                logger.error(format_exception(e))



                # Raise the exception if we are in debug mode

                if settings.DEBUG:

                    raise



                # Otherwise,

                # Otherwise, mask it

                else:



                    # first log the exception

                    logger.error(format_exception(e))



                    # and then mask it.

                    error_text = 'something went wrong'

                    error_text = 'something went wrong ({})'.format(e)



            data = {'user': request.user,

                    'title': 'Error',
@@ -111,18 +109,16 @@ def private_view(wrapped_view): 
                    error_text = str(e)

                else:



                    # Raise te exception if we are in debug mode

                    # Log the exception 

                    logger.error(format_exception(e))

    

                    # Raise the exception if we are in debug mode

                    if settings.DEBUG:

                        raise

    

                    # Otherwise,

                    # Otherwise, mask it

                    else:



                        # first log the exception

                        logger.error(format_exception(e))



                        # and then mask it.

                        error_text = 'something went wrong'

                        error_text = 'something went wrong ({})'.format(e)



                data = {'user': request.user,

                        'title': 'Error',

services/webapp/code/rosetta/settings.py

+2 −1
Original line number Diff line number Diff line
@@ -133,7 +133,8 @@ USE_TZ = True 
# Static files (CSS, JavaScript, Images)

# https://docs.djangoproject.com/en/2.2/howto/static-files/



STATIC_URL = '/static/'

STATIC_URL = '/static/'         # URL path

STATIC_ROOT = '/rosetta/static' # Filesystem path





# REST framework settings

services/webapp/requirements.txt

+1 −0
Original line number Diff line number Diff line
@@ -6,3 +6,4 @@ django-rest-swagger==2.2.0 
dateutils==0.6.6

sendgrid==5.3.0

mozilla-django-oidc==1.2.4

uwsgi==2.0.19.1