services/webapp/code/rosetta/core_app/templatetags/computing_helpers.py

+from django import template
+
+register = template.Library()
+
+@register.simple_tag
+def is_computing_configured(computing, user):
+    return computing.manager.is_configured_for(user)
+
+
+# {% is_computing_configured computing user %}

services/webapp/code/rosetta/core_app/utils.py

@@ -3,6 +3,8 @@ import re
 import hashlib
 import traceback
 import hashlib
+import json
+import requests
 import random
 import subprocess
 import logging
@@ -142,7 +144,7 @@ def finalize_user_creation(user, auth='local'):
     # Just an extra check
     try:
         Profile.objects.get(user=user)
-    except Profile.DoesNotExists:
+    except Profile.DoesNotExist:
         pass
     else:
         raise Exception('Consistency error: already found a profile for user "{}"'.format(user))
@@ -507,11 +509,11 @@ def get_webapp_conn_string():
     webapp_conn_string = 'http://{}:{}'.format(webapp_host, webapp_port)
     return webapp_conn_string
 
-def get_local_docker_registry_conn_string():
-    local_docker_registry_host = os.environ.get('LOCAL_DOCKER_REGISTRY_HOST', 'dregistry')
-    local_docker_registry_port = os.environ.get('LOCAL_DOCKER_REGISTRY_PORT', '5000')
-    local_docker_registry_conn_string = '{}:{}'.format(local_docker_registry_host, local_docker_registry_port)
-    return local_docker_registry_conn_string
+def get_platform_registry():
+    platform_registry_host = os.environ.get('PLATFORM_REGISTRY_HOST', 'proxy')
+    platform_registry_port = os.environ.get('PLATFORM_REGISTRY_PORT', '5000')
+    platform_registry_conn_string = '{}:{}'.format(platform_registry_host, platform_registry_port)
+    return platform_registry_conn_string
     
 def get_task_tunnel_host():
     tunnel_host = os.environ.get('TASK_TUNNEL_HOST', 'localhost')
@@ -713,13 +715,13 @@ def get_ssh_access_mode_credentials(computing, user):
     except AttributeError:
         computing_host = None
     if not computing_host:
-        raise Exception('No computing host?!')
+        raise ValueError('No computing host?!')
             
     # Get computing user and keys
     if computing.auth_mode == 'user_keys':
         computing_user = user.profile.get_extra_conf('computing_user', computing)
         if not computing_user:
-            raise Exception('Computing resource \'{}\' user is not configured'.format(computing.name))
+            raise ValueError('No \'computing_user\' parameter found for computing resource \'{}\' in user profile'.format(computing.name))
         # Get user key
         computing_keys = KeyPair.objects.get(user=user, default=True)
     elif computing.auth_mode == 'platform_keys':        
@@ -728,7 +730,7 @@ def get_ssh_access_mode_credentials(computing, user):
     else:
         raise NotImplementedError('Auth modes other than user_keys and platform_keys not supported.')
     if not computing_user:
-        raise Exception('No computing user?!')
+            raise ValueError('No \'user\' parameter found for computing resource \'{}\' in its configuration'.format(computing.name))
     return (computing_user, computing_host, computing_keys)
 
 
@@ -743,3 +745,116 @@ def sanitize_container_env_vars(env_vars):
 
     return env_vars
 
+
+def get_or_create_container_from_repository(user, repository_url, repository_tag=None, container_name=None, container_description=None): 
+    
+    from .models import Container    
+    logger.debug('Called get_or_create_container_from_repository with repository_url="{}" and repository_tag="{}"'.format(repository_url,repository_tag))
+
+    # Set repo name
+    repository_name = '{}/{}'.format(repository_url.split('/')[-2],repository_url.split('/')[-1])
+
+    # If building:
+    #{"message": "Successfully built 5a2089b2c334\n", "phase": "building"}
+    #{"message": "Successfully tagged r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3:latest\n", "phase": "building"}
+    
+    # If reusing:
+    #{"message": "Reusing existing image (r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3), not building."}
+    
+    appendix = 'CMD ["jupyter", "notebook", "--ip", "0.0.0.0", "--NotebookApp.token", ""]'
+
+    
+    # Build the Docker container for this repo
+    if repository_tag:
+        command = 'sudo jupyter-repo2docker --ref {} --user-id 1000 --user-name jovyan --no-run --appendix \'{}\' --json-logs {}'.format(repository_tag, appendix, repository_url)
+    else:
+        command = 'sudo jupyter-repo2docker --user-id 1000 --user-name jovyan --no-run --appendix \'{}\' --json-logs {}'.format(repository_url, appendix)
+    out = os_shell(command, capture=True)
+    if out.exit_code != 0:
+        logger.error(out.stderr)
+        raise ErrorMessage('Something went wrong when creating the Dockerfile for repository "{}"'.format(repository_url))   
+
+    # Convert output to lines
+    out_lines = out.stderr.split('\n')
+
+    # Get rep2docker image name from output. Use "strip()" as sometimes the newline chars might jump in.
+    last_line_message = json.loads(out_lines[-1])['message']
+    if 'Reusing existing image' in last_line_message:
+        repo2docker_image_name = last_line_message.split('(')[1].split(')')[0].strip()
+    elif 'Successfully tagged' in last_line_message:
+        repo2docker_image_name = last_line_message.split(' ')[2].strip()
+    else:
+        raise Exception('Cannot build')
+
+    # Set image registry, name and tag. Use "strip()" as sometimes the newline chars might jump in.
+    registry = get_platform_registry()
+    image_name = repository_name.lower().strip()
+    if repo2docker_image_name.endswith(':latest'):
+        # Not clear why sometimes this happens. maybe if an existent image gets reused?
+        image_name_for_tag = repo2docker_image_name.replace(':latest','')
+    else:
+        image_name_for_tag = repo2docker_image_name
+    
+    image_tag = image_name_for_tag[-7:] # The last part of the image name generated by repo2docker is the git short hash
+
+    # Re-tag image taking into account that if we are using the proxy as registry we use localhost or it won't work
+    if registry == 'proxy:5000':
+        push_registry = 'localhost:5000'
+    else:
+        push_registry = registry
+        
+    out = os_shell('sudo docker tag {} {}/{}:{}'.format(repo2docker_image_name,push_registry,image_name,image_tag) , capture=True)
+    if out.exit_code != 0:
+        logger.error(out.stderr)
+        raise ErrorMessage('Something went wrong when tagging the container for repository "{}"'.format(repository_url))   
+
+    # Push image to the (local) registry
+    out = os_shell('sudo docker push {}/{}:{}'.format(push_registry,image_name,image_tag) , capture=True)
+    if out.exit_code != 0:
+        logger.error(out.stderr)
+        raise ErrorMessage('Something went wrong when pushing the container for repository "{}"'.format(repository_url))   
+
+    # Create the container if not already existent
+    try:
+        container = Container.objects.get(user=user, registry=registry, image_name=image_name, image_tag=image_tag)
+    except Container.DoesNotExist:
+
+        # Get name repo name and description from remote if we have and if we can
+        repository_name_from_source= None
+        repository_description_from_source = None
+        if not container_name or not container_description:
+            if repository_url.startswith('https://github.com'):
+                try:
+                    response = requests.get('https://api.github.com/repos/{}'.format(repository_name))
+                    json_content = json.loads(response.content)
+                    repository_name_from_source = json_content['name'].title()
+                    repository_description_from_source = json_content['description']
+                    if not repository_description_from_source.endswith('.'):
+                        repository_description_from_source+='.'
+                    repository_description_from_source += ' Built from {}'.format(repository_url)
+                except:
+                    pass
+        
+        # Set default container name and description            
+        if not container_name:
+            container_name = repository_name_from_source if repository_name_from_source else repository_name
+        
+        if not container_description:
+            container_description = repository_description_from_source if repository_description_from_source else 'Built from {}'.format(repository_url)
+
+        # Ok, create the container
+        container = Container.objects.create(user = user,
+                                             name = container_name,
+                                             description = container_description,
+                                             registry = registry,
+                                             image_name = image_name,
+                                             image_tag  = image_tag,
+                                             image_arch = 'amd64',
+                                             image_os = 'linux',
+                                             interface_port = '8888',
+                                             interface_protocol = 'http',
+                                             interface_transport = 'tcp/ip',
+                                             supports_custom_interface_port = False,
+                                             supports_interface_auth = False)
+    return container
+

services/webapp/code/rosetta/core_app/views.py

 import os
 import uuid
 import json
+import requests
+import socket
 import subprocess
 import base64
 from django.conf import settings
@@ -11,7 +13,9 @@ from django.contrib.auth.models import User
 from django.shortcuts import redirect
 from django.db.models import Q
 from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, Page
-from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_task_tunnel_host, get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation, sanitize_container_env_vars
+from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_task_tunnel_host
+from .utils import get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation
+from .utils import sanitize_container_env_vars, get_or_create_container_from_repository
 from .decorators import public_view, private_view
 from .exceptions import ErrorMessage
 
@@ -32,15 +36,21 @@ _task_cache = {}
 def login_view(request):
 
     data = {}
+    
+    # Set post login page
+    post_login_page = request.COOKIES.get('post_login_redirect')
+    if post_login_page is None:
+        post_login_page = '/main'
 
     # If authenticated user reloads the main URL
     if request.method == 'GET' and request.user.is_authenticated:
-        return HttpResponseRedirect('/main/')
+        response = HttpResponseRedirect(post_login_page)
+        response.delete_cookie('post_login_redirect')
+        return response
     else:
-        # If local auth disabled, just render login page
-        # (will be rendered an open id connect url only)
+        # If local auth disabled, just redirect to OIDC
         if settings.DISABLE_LOCAL_AUTH:
-            return render(request, 'login.html', {'data': data})
+            return HttpResponseRedirect('/oidc/authenticate/')
 
     # If unauthenticated user tries to log in
     if request.method == 'POST':
@@ -70,7 +80,9 @@ def login_view(request):
                 user = authenticate(username=username, password=password)
                 if user:
                     login(request, user)
-                    return HttpResponseRedirect('/main')
+                    response = HttpResponseRedirect(post_login_page)
+                    response.delete_cookie('post_login_redirect')
+                    return response
                 else:
                     raise ErrorMessage('Check email and password')
             else:
@@ -135,8 +147,9 @@ def login_view(request):
             loginToken.delete()
 
             # Now redirect to site
-            return HttpResponseRedirect('/main/')
-
+            response = HttpResponseRedirect(post_login_page)
+            response.delete_cookie('post_login_redirect')
+            return response
 
     # All other cases, render the login page again with no other data than title
     return render(request, 'login.html', {'data': data})
@@ -329,6 +342,30 @@ def account(request):
 #  Tasks view
 #=========================
 
+def set_verified_status(task):
+    # Chech status with ping
+    if task.status == 'running':
+        logger.debug('Task is running, check if startup completed')
+
+        logger.debug('Trying to establish connection on: "{}:{}"'.format(task.interface_ip,task.interface_port))
+        s = socket.socket()
+        try:
+            s.settimeout(1)
+            s.connect((task.interface_ip, task.interface_port))
+            # Not necessary, we just check that the container interfcae is up
+            #if not s.recv(10):
+            #    logger.debug('No data read from socket')
+            #    raise Exception('Could not read any data from socket')
+        except Exception as e:
+            logger.debug('Could not connect to socket')
+            task.verified_status = 'starting up...'
+        else:
+            task.verified_status = 'running'
+        finally:
+            s.close()
+    else:
+        task.verified_status = task.status
+
 @private_view
 def tasks(request):
 
@@ -355,8 +392,10 @@ def tasks(request):
                 task = Task.objects.get(user=request.user, uuid=uuid)
             except Task.DoesNotExist:
                 raise ErrorMessage('Task does not exists or no access rights')
+            
+            set_verified_status(task)
             data['task'] = task
-    
+            
             #  Task actions
             if action=='delete':
                 if task.status not in [TaskStatuses.stopped, TaskStatuses.exited]:
@@ -435,7 +474,8 @@ def tasks(request):
         # Update task statuses
         for task in tasks:
             task.update_status()
-    
+            set_verified_status(task)
+        
         # Set task and tasks variables
         data['task']  = None   
         data['tasks'] = tasks
@@ -897,89 +937,112 @@ def add_software(request):
     # Init data
     data = {}
     data['user'] = request.user
+    
+    # Loop back the new container mode in the page to handle the switch
+    data['new_container_from'] = request.GET.get('new_container_from', 'registry')
 
     # Container name if setting up a new container
     container_name = request.POST.get('container_name', None)
 
     if container_name:
-
-        # Container description
-        container_description = request.POST.get('container_description', None)
-
-        # Container registry
-        container_registry = request.POST.get('container_registry', None)
-
-        # Container image name
-        container_image_name = request.POST.get('container_image_name',None)
         
-        # Container image tag
-        container_image_tag = request.POST.get('container_image_tag', None)
+        # How do we have to add this new container?
+        new_container_from = request.POST.get('new_container_from', None)
 
-        # Container image architecture
-        container_image_arch = request.POST.get('container_image_arch', None)
-
-        # Container image OS 
-        container_image_os = request.POST.get('container_image_os', None)
-
-        # Container image digest
-        container_image_digest = request.POST.get('container_image_digest', None)
-
-        # Container interface port
-        container_interface_port = request.POST.get('container_interface_port', None) 
-        if container_interface_port:       
-            try:
-                container_interface_port = int(container_interface_port)
-            except:
-                raise ErrorMessage('Invalid container port "{}"')
-        else:
-            container_interface_port = None
-
-        # Container interface protocol 
-        container_interface_protocol = request.POST.get('container_interface_protocol', None)
-
-        if container_interface_protocol and not container_interface_protocol in ['http','https']:
-            raise ErrorMessage('Sorry, only power users can add custom software containers with interface protocols other than \'http\' or \'https\'.')
+        if new_container_from == 'registry':
+    
+            # Container description
+            container_description = request.POST.get('container_description', None)
+    
+            # Container registry
+            container_registry = request.POST.get('container_registry', None)
+    
+            # Container image name
+            container_image_name = request.POST.get('container_image_name',None)
+            
+            # Container image tag
+            container_image_tag = request.POST.get('container_image_tag', None)
+    
+            # Container image architecture
+            container_image_arch = request.POST.get('container_image_arch', None)
+    
+            # Container image OS 
+            container_image_os = request.POST.get('container_image_os', None)
+    
+            # Container image digest
+            container_image_digest = request.POST.get('container_image_digest', None)
+    
+            # Container interface port
+            container_interface_port = request.POST.get('container_interface_port', None) 
+            if container_interface_port:       
+                try:
+                    container_interface_port = int(container_interface_port)
+                except:
+                    raise ErrorMessage('Invalid container port "{}"')
+            else:
+                container_interface_port = None
+    
+            # Container interface protocol 
+            container_interface_protocol = request.POST.get('container_interface_protocol', None)
+    
+            if container_interface_protocol and not container_interface_protocol in ['http','https']:
+                raise ErrorMessage('Sorry, only power users can add custom software containers with interface protocols other than \'http\' or \'https\'.')
+    
+            # Container interface transport 
+            container_interface_transport = request.POST.get('container_interface_transport')
+    
+            # Capabilities
+            container_supports_custom_interface_port = request.POST.get('container_supports_custom_interface_port', None)
+            if container_supports_custom_interface_port and container_supports_custom_interface_port == 'True':
+                container_supports_custom_interface_port = True
+            else:
+                container_supports_custom_interface_port = False
+    
+            container_supports_interface_auth = request.POST.get('container_supports_interface_auth', None)
+            if container_supports_interface_auth and container_supports_interface_auth == 'True':
+                container_supports_pass_auth = True
+            else:
+                container_supports_pass_auth = False
+    
+            # Environment variables
+            container_env_vars = request.POST.get('container_env_vars', None)
+            if container_env_vars:
+                container_env_vars = sanitize_container_env_vars(json.loads(container_env_vars))
+    
+            # Log
+            #logger.debug('Creating new container object with image="{}", type="{}", registry="{}", ports="{}"'.format(container_image, container_type, container_registry, container_ports))
+    
+            # Create
+            Container.objects.create(user         = request.user,
+                                     name         = container_name,
+                                     description  = container_description,
+                                     registry     = container_registry,
+                                     image_name   = container_image_name,
+                                     image_tag    = container_image_tag,
+                                     image_arch   = container_image_arch,
+                                     image_os     = container_image_os,
+                                     image_digest = container_image_digest,
+                                     interface_port      = container_interface_port,
+                                     interface_protocol  = container_interface_protocol,
+                                     interface_transport = container_interface_transport,
+                                     supports_custom_interface_port = container_supports_custom_interface_port,
+                                     supports_interface_auth = container_supports_pass_auth,
+                                     env_vars = container_env_vars)
+            
+        elif new_container_from == 'repository':
 
-        # Container interface transport 
-        container_interface_transport = request.POST.get('container_interface_transport')
+            container_description = request.POST.get('container_description', None)
+            
+            repository_url = request.POST.get('repository_url', None)
+            repository_tag = request.POST.get('repository_tag', 'HEAD')
 
-        # Capabilities
-        container_supports_custom_interface_port = request.POST.get('container_supports_custom_interface_port', None)
-        if container_supports_custom_interface_port and container_supports_custom_interface_port == 'True':
-            container_supports_custom_interface_port = True
-        else:
-            container_supports_custom_interface_port = False
+            return HttpResponseRedirect('/import_repository/?repository_url={}&repository_tag={}&container_name={}&container_description={}'.format(repository_url,repository_tag,container_name,container_description))
+        
+            # The return type here is a container, not created
+            #get_or_create_container_from_repository(request.user, repository_url, repository_tag=repository_tag, container_name=container_name, container_description=container_description)
 
-        container_supports_interface_auth = request.POST.get('container_supports_interface_auth', None)
-        if container_supports_interface_auth and container_supports_interface_auth == 'True':
-            container_supports_pass_auth = True
         else:
-            container_supports_pass_auth = False
-
-        # Environment variables
-        container_env_vars = request.POST.get('container_env_vars', None)
-        if container_env_vars:
-            container_env_vars = sanitize_container_env_vars(json.loads(container_env_vars))
-
-        # Log
-        #logger.debug('Creating new container object with image="{}", type="{}", registry="{}", ports="{}"'.format(container_image, container_type, container_registry, container_ports))
-
-        # Create
-        Container.objects.create(user         = request.user,
-                                 name         = container_name,
-                                 description  = container_description,
-                                 registry     = container_registry,
-                                 image_name   = container_image_name,
-                                 image_tag    = container_image_tag,
-                                 image_arch   = container_image_arch,
-                                 image_os     = container_image_os,
-                                 image_digest = container_image_digest,
-                                 interface_port      = container_interface_port,
-                                 interface_protocol  = container_interface_protocol,
-                                 interface_transport = container_interface_transport,
-                                 supports_custom_interface_port = container_supports_custom_interface_port,
-                                 supports_interface_auth = container_supports_pass_auth,
-                                 env_vars = container_env_vars)
+            raise Exception('Unknown new container mode "{}"'.format(new_container_from)) 
         # Set added switch
         data['added'] = True
 
@@ -1166,14 +1229,72 @@ def sharable_link_handler(request, short_uuid):
     return redirect(redirect_string)
 
 
+#=========================
+#  New Binder Task
+#=========================
 
+@private_view
+def new_binder_task(request, repository):
 
+    # Init data
+    data={}
+    data['user']  = request.user
 
+    # Convert the Git repository as a Docker container
+    logger.debug('Got a new Binder task request for repository "%s"', repository)
+        
+    # Set repository name/tag/url        
+    repository_tag = repository.split('/')[-1]
+    repository_url = repository.replace('/'+repository_tag, '')
+
+    # I have no idea why the https:// of the repo part of the url gets transfrmed in https:/
+    # Here i work around this, but TODO: understand what the hell is going on.
+    if 'https:/' in repository_url and not 'https://' in repository_url:
+        repository_url = repository_url.replace('https:/', 'https://')
+    
+    if not repository_tag:
+        repository_tag='HEAD'
 
+    data['repository_url'] = repository_url
+    data['repository_tag'] = repository_tag
+    
+    data['mode'] = 'new_task' #new container
+
+    # Render the import page. This will call an API, and when the import is done, it
+    # will automatically redirect to the page "new_task/?step=two&task_container_uuid=..."
+    return render(request, 'import_repository.html', {'data': data})
 
 
+#=========================
+#  Import repository
+#=========================
 
+@private_view
+def import_repository(request):
+
+    # Init data
+    data={}
+    data['user']  = request.user
+    
+    repository_url = request.GET.get('repository_url', None)
+    # I have no idea why the https:// of the repo part of the url gets transfrmed in https:/
+    # Here i work around this, but TODO: understand what the hell is going on.
+    if 'https:/' in repository_url and not 'https://' in repository_url:
+        repository_url = repository_url.replace('https:/', 'https://')
+    
+    repository_tag= request.GET.get('repository_tag', None)
+    if not repository_tag:
+        repository_tag='HEAD'
+        
+    data['repository_url'] = repository_url
+    data['repository_tag'] = repository_tag
 
+    data['container_name'] = request.GET.get('container_name', None)
+    data['container_description'] = request.GET.get('container_description', None)
 
+    data['mode'] = 'new_container'
 
+    # Render the import page. This will call an API, and when the import is done, it
+    # will automatically say "Ok, crrated, go to software".
+    return render(request, 'import_repository.html', {'data': data})
 

services/webapp/code/rosetta/settings.py

@@ -263,7 +263,10 @@ if OIDC_RP_CLIENT_ID:
 
     # Optional
     OIDC_USE_NONCE =  booleanize(os.environ.get('OIDC_USE_NONCE', False))
-    OIDC_TOKEN_USE_BASIC_AUTH = booleanize(os.environ.get('OIDC_TOKEN_USE_BASIC_AUTH', True))
+    OIDC_TOKEN_USE_BASIC_AUTH = booleanize(os.environ.get('OIDC_TOKEN_USE_BASIC_AUTH', False))
+
+    # Custom callback to enable session-based post-login redirects
+    OIDC_CALLBACK_CLASS = 'rosetta.auth.RosettaOIDCAuthenticationCallbackView'
     
     # Non-customizable stuff
     LOGIN_REDIRECT_URL = '/'
@@ -274,6 +277,8 @@ if OIDC_RP_CLIENT_ID:
     # Required for the Open ID connect redirects to work properly
     USE_X_FORWARDED_HOST = True
     SECURE_PROXY_SSL_HEADER = ('HTTP_X_FORWARDED_PROTO', 'https')
+    
+
 
 
 

services/webapp/code/rosetta/urls.py

@@ -51,6 +51,7 @@ urlpatterns = [
     # Software    
     url(r'^software/$', core_app_views.software),
     url(r'^add_software/$', core_app_views.add_software),    
+    url(r'^import_repository/$', core_app_views.import_repository),    
     
     #Computing
     url(r'^computing/$', core_app_views.computing),
@@ -84,6 +85,11 @@ urlpatterns = [
     path('api/v1/base/logout/', core_app_api.logout_api.as_view(), name='logout_api'),
     path('api/v1/base/agent/', core_app_api.agent_api.as_view(), name='agent_api'),
     path('api/v1/filemanager/', core_app_api.FileManagerAPI.as_view(), name='filemanager_api'),
+    path('api/v1/import_repository/', core_app_api.ImportRepositoryAPI.as_view(), name='import_repository_api'),
+
+    # Binder compatibility
+    path('v2/git/<path:repository>', core_app_views.new_binder_task),
+
 
 ]
 

services/webapp/requirements.txt

@@ -8,3 +8,4 @@ sendgrid==5.3.0
 mozilla-django-oidc==1.2.4
 uwsgi==2.0.19.1
 python-magic==0.4.15
+jupyter-repo2docker==2022.2.0