Merge branch 'feature/container_from_repo' into develop (99f7495b) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/templates/add_software.html

+84 −11

Original line number	Diff line number	Diff line
		@@ -13,12 +13,23 @@

		{% if not data.added %}

		<h4>Container basics</h4>
		Here you can add a new software container on the platform. You can add containers from image registries
		as <a href="https://hub.docker.com/">Docker Hub </a>or by importing Git repositories, provided that they
		are compatible with <a href="https://mybinder.readthedocs.io/en/latest/introduction.html">Binder</a> specifications.
		<br/>
		<br/>

		{% if data.new_container_from == 'registry' %}
		<div style="font-size:1.2em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from registry</div>
		<div style="font-size:1.2em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=repository">New container from Git repository</a></div>
		<hr style="margin-top:0;">

		<h4>Basics</h4>

		<form action="#" method="POST">
		{% csrf_token %}

		<table class="dashboard" style="width:360px; margin-bottom:25px">
		<table class="dashboard" style="width:400px; margin-bottom:25px">

		<tr>
		<td><b>Name</b></td>
		@@ -35,8 +46,6 @@
		</td>
		</tr>



		<tr>
		<td><b>Registry</b></td><td>
		<input type="text" name="container_registry" value="docker.io" size="23" required />
		@@ -61,8 +70,8 @@



		<h4>Container interface </h4>
		<table class="dashboard" style="width:360px; margin-bottom:25px">
		<h4>Interface </h4>
		<table class="dashboard" style="width:400px; margin-bottom:25px">

		<tr>
		<td><b>Interface port</b></td>
		@@ -88,10 +97,10 @@
		<a href="javascript:void(0);" id="show_button" onclick="toggle_visibility('advanced_div')">Advanced...</a>


		<div id="advanced_div" style="display:none; width:360px;">
		<h4>Container advanced settings <font size=-1>\| <a href="javascript:void(0);" id="hide_button" onclick="toggle_visibility('advanced_div')" style="display:none">hide</a></font></h4>
		<div id="advanced_div" style="display:none; width:400px;">
		<h4>Advanced <font size=-1>\| <a href="javascript:void(0);" id="hide_button" onclick="toggle_visibility('advanced_div')" style="display:none">hide</a></font></h4>

		<table class="dashboard" style="width:360px; margin-bottom:25px">
		<table class="dashboard" style="width:400px; margin-bottom:25px">

		<tr>
		<td><b>Image arch</b></td><td>
		@@ -145,14 +154,78 @@
		</table>
		</div>

		<table style="width:360px; border:0; background:#ffffff; margin-top:20px">
		<table style="width:400px; border:0; background:#ffffff; margin-top:20px">
		<tr><td align="center">
		<input type="submit" value="Add">
		</td></tr>
		</table>

		<input type="hidden" name="new_container_from" value="registry">

		</form>


		{% else %}
		<div style="font-size:1.2em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=registry">New container from registry</a></div>
		<div style="font-size:1.2em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from Git repository</div>
		<hr style="margin-top:0;">

		<h4>Basics</h4>

		<form action="#" method="POST">
		{% csrf_token %}

		<table class="dashboard" style="width:400px; margin-bottom:25px">

		<tr>
		<td><b>Name</b></td>
		<td>
		<input type="text" name="container_name" value="" placeholder="" size="23" required />
		</td>
		</tr>

		<tr>
		<td><b>Description</b></td>
		<td>
		<!-- ><input type="text" name="container_description" value="" placeholder="" size="23" required /> -->
		<textarea name="container_description" rows="3" cols="22"></textarea>
		</td>
		</tr>

		<tr>
		<td><b>Repository URL</b></td><td>
		<input type="text" name="repository_url" size="23" required />
		</td>
		</tr>


		<tr>
		<td><b>Repository tag</b></td>
		<td>
		<input type="text" name="repository_tag" placeholder="Tag or hash, optional" value="" size="23" />
		</td>
		</tr>

		</table>



		<table style="width:400px; border:0; background:#ffffff; margin-top:20px">
		<tr><td align="center">
		<input type="submit" value="Add">
		</td></tr>
		</table>

		<input type="hidden" name="new_container_from" value="repository">

		</form>



		{% endif %}



		<br/>
		<br/>
		<br/>

services/webapp/code/rosetta/core_app/utils.py

+106 −0

Original line number	Diff line number	Diff line
		@@ -3,6 +3,8 @@ import re
		import hashlib
		import traceback
		import hashlib
		import json
		import requests
		import random
		import subprocess
		import logging
		@@ -743,3 +745,107 @@ def sanitize_container_env_vars(env_vars):

		return env_vars


		def get_or_create_container_from_repository(user, repository_url, repository_tag=None, container_name=None, container_description=None):

		from .models import Container
		logger.debug('Called get_or_create_container_from_repository with repository_url="{}" and repository_tag="{}"'.format(repository_url,repository_tag))

		# Set repo name
		repository_name = '{}/{}'.format(repository_url.split('/')[-2],repository_url.split('/')[-1])

		# If building:
		#{"message": "Successfully built 5a2089b2c334\n", "phase": "building"}
		#{"message": "Successfully tagged r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3:latest\n", "phase": "building"}

		# If reusing:
		#{"message": "Reusing existing image (r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3), not building."}

		# Build the Docker container for this repo
		if repository_tag:
		command = 'sudo jupyter-repo2docker --ref {} --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_tag, repository_url)
		else:
		command = 'sudo jupyter-repo2docker --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_url)
		out = os_shell(command, capture=True)
		if out.exit_code != 0:
		logger.error(out.stderr)
		raise ErrorMessage('Something went wrong when creating the Dockerfile for repository "{}"'.format(repository_url))

		# Convert output to lines
		out_lines = out.stderr.split('\n')

		# Get rep2docker image name from output. Use "strip()" as sometimes the newline chars might jump in.
		last_line_message = json.loads(out_lines[-1])['message']
		if 'Reusing existing image' in last_line_message:
		repo2docker_image_name = last_line_message.split('(')[1].split(')')[0].strip()
		elif 'Successfully tagged' in last_line_message:
		repo2docker_image_name = last_line_message.split(' ')[2].strip()
		else:
		raise Exception('Cannot build')

		# Set image registry, name and tag. Use "strip()" as sometimes the newline chars might jump in.
		registry = os.environ.get('REGISTRY_HOST','proxy:5000').strip()
		image_name = repository_name.lower().strip()
		image_tag = repo2docker_image_name[-7:] # The last part of the image name generated by repo2docker is the git short hash

		# Re-tag image taking into account that if we are using the proxy as registry we use localhost or it won't work
		if registry == 'proxy:5000':
		push_registry = 'localhost:5000'
		else:
		push_registry = registry

		out = os_shell('sudo docker tag {} {}/{}:{}'.format(repo2docker_image_name,push_registry,image_name,image_tag) , capture=True)
		if out.exit_code != 0:
		logger.error(out.stderr)
		raise ErrorMessage('Something went wrong when tagging the container for repository "{}"'.format(repository_url))

		# Push image to the (local) registry
		out = os_shell('sudo docker push {}/{}:{}'.format(push_registry,image_name,image_tag) , capture=True)
		if out.exit_code != 0:
		logger.error(out.stderr)
		raise ErrorMessage('Something went wrong when pushing the container for repository "{}"'.format(repository_url))

		# Create the container if not already existent
		try:
		container = Container.objects.get(user=user, registry=registry, image_name=image_name, image_tag=image_tag)
		except Container.DoesNotExist:

		# Get name repo name and description from remote if we have and if we can
		repository_name_from_source= None
		repository_description_from_source = None
		if not container_name or not container_description:
		if repository_url.startswith('https://github.com'):
		try:
		response = requests.get('https://api.github.com/repos/{}'.format(repository_name))
		json_content = json.loads(response.content)
		repository_name_from_source = json_content['name'].title()
		repository_description_from_source = json_content['description']
		if not repository_description_from_source.endswith('.'):
		repository_description_from_source+='.'
		repository_description_from_source += ' Built from {}'.format(repository_url)
		except:
		pass

		# Set default container name and description
		if not container_name:
		container_name = repository_name_from_source if repository_name_from_source else repository_name

		if not container_description:
		container_description = repository_description_from_source if repository_description_from_source else 'Built from {}'.format(repository_url)

		# Ok, create the container
		container = Container.objects.create(user = user,
		name = container_name,
		description = container_description,
		registry = registry,
		image_name = image_name,
		image_tag = image_tag,
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		supports_custom_interface_port = False,
		supports_interface_auth = False)
		return container

services/webapp/code/rosetta/core_app/views.py

+97 −170

Original line number	Diff line number	Diff line
		@@ -12,7 +12,9 @@ from django.contrib.auth.models import User
		from django.shortcuts import redirect
		from django.db.models import Q
		from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, Page
		from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_task_tunnel_host, get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation, sanitize_container_env_vars
		from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_task_tunnel_host
		from .utils import get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation
		from .utils import sanitize_container_env_vars, get_or_create_container_from_repository
		from .decorators import public_view, private_view
		from .exceptions import ErrorMessage

		@@ -899,11 +901,19 @@ def add_software(request):
		data = {}
		data['user'] = request.user

		# Loop back the new container mode in the page to handle the switch
		data['new_container_from'] = request.GET.get('new_container_from', 'registry')

		# Container name if setting up a new container
		container_name = request.POST.get('container_name', None)

		if container_name:

		# How do we have to add this new container?
		new_container_from = request.POST.get('new_container_from', None)

		if new_container_from == 'registry':

		# Container description
		container_description = request.POST.get('container_description', None)

		@@ -981,6 +991,18 @@ def add_software(request):
		supports_custom_interface_port = container_supports_custom_interface_port,
		supports_interface_auth = container_supports_pass_auth,
		env_vars = container_env_vars)

		elif new_container_from == 'repository':

		container_description = request.POST.get('container_description', None)
		repository_url = request.POST.get('repository_url', None)
		repository_tag = request.POST.get('repository_tag', None)

		# The return type here is a container, not created
		get_or_create_container_from_repository(request.user, repository_url, repository_tag=repository_tag, container_name=container_name, container_description=container_description)

		else:
		raise Exception('Unknown new container mode "{}"'.format(new_container_from))
		# Set added switch
		data['added'] = True

		@@ -1167,101 +1189,6 @@ def sharable_link_handler(request, short_uuid):
		return redirect(redirect_string)


		def get_or_create_container_from_repository(repository_url, repository_tag=None):
		repository_name = '{}/{}'.format(repository_url.split('/')[-2],repository_url.split('/')[-1])

		logger.debug('Called get_or_create_container_from_repository with repository_url="{}" and repository_tag="{}"'.format(repository_url,repository_tag))

		# If building:
		#{"message": "Successfully built 5a2089b2c334\n", "phase": "building"}
		#{"message": "Successfully tagged r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3:latest\n", "phase": "building"}

		# If reusing:
		#{"message": "Reusing existing image (r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3), not building."}

		# Build the Docker container for this repo
		if repository_tag:
		command = 'sudo jupyter-repo2docker --ref {} --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_tag, repository_url)
		else:
		command = 'sudo jupyter-repo2docker --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_url)
		out = os_shell(command, capture=True)
		if out.exit_code != 0:
		logger.error(out.stderr)
		raise ErrorMessage('Something went wrong when creating the Dockerfile for repository "{}"'.format(repository_url))

		# Convert output to lines
		out_lines = out.stderr.split('\n')

		# Get rep2docker image name from output
		last_line_message = json.loads(out_lines[-1])['message']
		if 'Reusing existing image' in last_line_message:
		repo2docker_image_name = last_line_message.split('(')[1].split(')')[0]
		elif 'Successfully tagged' in last_line_message:
		repo2docker_image_name = last_line_message.split(' ')[2]
		else:
		raise Exception('Cannot build')

		# Set image registry, name and tag, Use "strip()" as sometimes the newline chars might jump in.
		registry = os.environ.get('REGISTRY_HOST','proxy:5000').strip()
		image_name = repository_name.lower().strip()
		image_tag = repo2docker_image_name[-7:].strip() # The last part of the image name generated by repo2docker is the git short hash

		# Re-tag image taking into account that if we are using the proxy as registry we use localhost or it won't work
		if registry == 'proxy:5000':
		push_registry = 'localhost:5000'
		else:
		push_registry = registry

		out = os_shell('sudo docker tag {} {}/{}:{}'.format(repo2docker_image_name,push_registry,image_name,image_tag) , capture=True)
		if out.exit_code != 0:
		logger.error(out.stderr)
		raise ErrorMessage('Something went wrong when tagging the container for repository "{}"'.format(repository_url))

		# Push image to the (local) registry
		out = os_shell('sudo docker push {}/{}:{}'.format(push_registry,image_name,image_tag) , capture=True)
		if out.exit_code != 0:
		logger.error(out.stderr)
		raise ErrorMessage('Something went wrong when pushing the container for repository "{}"'.format(repository_url))

		# Create the container if not already existent
		try:
		container = Container.objects.get(registry=registry, image_name=image_name, image_tag=image_tag)
		except Container.DoesNotExist:

		# Set default container name and description
		container_name = repository_name
		container_description = 'Built from {}'.format(repository_url)

		# Get name repo name and description from GitHub (if repo is there)
		if repository_url.startswith('https://github.com'):
		try:
		response = requests.get('https://api.github.com/repos/{}'.format(repository_name))
		json_content = json.loads(response.content)
		container_name = json_content['name'].title()
		container_description = json_content['description']
		if not container_description.endswith('.'):
		container_description+='.'
		container_description += ' Built from {}'.format(repository_url)
		except:
		pass

		container = Container.objects.create(user = None,
		name = container_name,
		description = container_description,
		registry = registry,
		image_name = image_name,
		image_tag = image_tag,
		image_arch = 'amd64',
		image_os = 'linux',
		interface_port = '8888',
		interface_protocol = 'http',
		interface_transport = 'tcp/ip',
		supports_custom_interface_port = False,
		supports_interface_auth = False)
		return container



		#=========================
		# New Binder Task
		#=========================
		@@ -1280,7 +1207,7 @@ def new_binder_task(request, repository):
		repository_tag = repository.split('/')[-1]
		repository_url = repository.replace('/'+repository_tag, '')

		container = get_or_create_container_from_repository(repository_url, repository_tag)
		container = get_or_create_container_from_repository(request.user, repository_url, repository_tag)

		# Set the container
		data['task_container'] = container