Merge branch 'feature/container_from_repo' into develop (99f7495b) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/templates/add_software.html

+84 −11

Original line number	Original line	Diff line number	Diff line
	@@ -13,12 +13,23 @@

	{% if not data.added %}		{% if not data.added %}

	<h4>Container basics</h4>		Here you can add a new software container on the platform. You can add containers from image registries
			as <a href="https://hub.docker.com/">Docker Hub </a>or by importing Git repositories, provided that they
			are compatible with <a href="https://mybinder.readthedocs.io/en/latest/introduction.html">Binder</a> specifications.
			<br/>
			<br/>

			{% if data.new_container_from == 'registry' %}
			<div style="font-size:1.2em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from registry</div>
			<div style="font-size:1.2em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=repository">New container from Git repository</a></div>
			<hr style="margin-top:0;">

			<h4>Basics</h4>

	<form action="#" method="POST">		<form action="#" method="POST">
	{% csrf_token %}		{% csrf_token %}

	<table class="dashboard" style="width:360px; margin-bottom:25px">		<table class="dashboard" style="width:400px; margin-bottom:25px">

	<tr>		<tr>
	<td><b>Name</b></td>		<td><b>Name</b></td>
	@@ -35,8 +46,6 @@
	</td>		</td>
	</tr>		</tr>



	<tr>		<tr>
	<td><b>Registry</b></td><td>		<td><b>Registry</b></td><td>
	<input type="text" name="container_registry" value="docker.io" size="23" required />		<input type="text" name="container_registry" value="docker.io" size="23" required />
	@@ -61,8 +70,8 @@



	<h4>Container interface </h4>		<h4>Interface </h4>
	<table class="dashboard" style="width:360px; margin-bottom:25px">		<table class="dashboard" style="width:400px; margin-bottom:25px">

	<tr>		<tr>
	<td><b>Interface port</b></td>		<td><b>Interface port</b></td>
	@@ -88,10 +97,10 @@
	<a href="javascript:void(0);" id="show_button" onclick="toggle_visibility('advanced_div')">Advanced...</a>		<a href="javascript:void(0);" id="show_button" onclick="toggle_visibility('advanced_div')">Advanced...</a>


	<div id="advanced_div" style="display:none; width:360px;">		<div id="advanced_div" style="display:none; width:400px;">
	<h4>Container advanced settings <font size=-1>\| <a href="javascript:void(0);" id="hide_button" onclick="toggle_visibility('advanced_div')" style="display:none">hide</a></font></h4>		<h4>Advanced <font size=-1>\| <a href="javascript:void(0);" id="hide_button" onclick="toggle_visibility('advanced_div')" style="display:none">hide</a></font></h4>

	<table class="dashboard" style="width:360px; margin-bottom:25px">		<table class="dashboard" style="width:400px; margin-bottom:25px">

	<tr>		<tr>
	<td><b>Image arch</b></td><td>		<td><b>Image arch</b></td><td>
	@@ -145,14 +154,78 @@
	</table>		</table>
	</div>		</div>

	<table style="width:360px; border:0; background:#ffffff; margin-top:20px">		<table style="width:400px; border:0; background:#ffffff; margin-top:20px">
			<tr><td align="center">
			<input type="submit" value="Add">
			</td></tr>
			</table>

			<input type="hidden" name="new_container_from" value="registry">

			</form>


			{% else %}
			<div style="font-size:1.2em; background:white; display:inline-block; padding:2px 15px 2px 15px"><a href="?new_container_from=registry">New container from registry</a></div>
			<div style="font-size:1.2em; background:whitesmoke; display:inline-block; padding:2px 15px 2px 15px">New container from Git repository</div>
			<hr style="margin-top:0;">

			<h4>Basics</h4>

			<form action="#" method="POST">
			{% csrf_token %}

			<table class="dashboard" style="width:400px; margin-bottom:25px">

			<tr>
			<td><b>Name</b></td>
			<td>
			<input type="text" name="container_name" value="" placeholder="" size="23" required />
			</td>
			</tr>

			<tr>
			<td><b>Description</b></td>
			<td>
			<!-- ><input type="text" name="container_description" value="" placeholder="" size="23" required /> -->
			<textarea name="container_description" rows="3" cols="22"></textarea>
			</td>
			</tr>

			<tr>
			<td><b>Repository URL</b></td><td>
			<input type="text" name="repository_url" size="23" required />
			</td>
			</tr>


			<tr>
			<td><b>Repository tag</b></td>
			<td>
			<input type="text" name="repository_tag" placeholder="Tag or hash, optional" value="" size="23" />
			</td>
			</tr>

			</table>



			<table style="width:400px; border:0; background:#ffffff; margin-top:20px">
	<tr><td align="center">		<tr><td align="center">
	<input type="submit" value="Add">		<input type="submit" value="Add">
	</td></tr>		</td></tr>
	</table>		</table>

			<input type="hidden" name="new_container_from" value="repository">

	</form>		</form>



			{% endif %}



	<br/>		<br/>
	<br/>		<br/>
	<br/>		<br/>

services/webapp/code/rosetta/core_app/utils.py

+106 −0

Original line number	Original line	Diff line number	Diff line
	@@ -3,6 +3,8 @@ import re
	import hashlib		import hashlib
	import traceback		import traceback
	import hashlib		import hashlib
			import json
			import requests
	import random		import random
	import subprocess		import subprocess
	import logging		import logging
	@@ -743,3 +745,107 @@ def sanitize_container_env_vars(env_vars):

	return env_vars		return env_vars


			def get_or_create_container_from_repository(user, repository_url, repository_tag=None, container_name=None, container_description=None):

			from .models import Container
			logger.debug('Called get_or_create_container_from_repository with repository_url="{}" and repository_tag="{}"'.format(repository_url,repository_tag))

			# Set repo name
			repository_name = '{}/{}'.format(repository_url.split('/')[-2],repository_url.split('/')[-1])

			# If building:
			#{"message": "Successfully built 5a2089b2c334\n", "phase": "building"}
			#{"message": "Successfully tagged r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3:latest\n", "phase": "building"}

			# If reusing:
			#{"message": "Reusing existing image (r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3), not building."}

			# Build the Docker container for this repo
			if repository_tag:
			command = 'sudo jupyter-repo2docker --ref {} --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_tag, repository_url)
			else:
			command = 'sudo jupyter-repo2docker --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_url)
			out = os_shell(command, capture=True)
			if out.exit_code != 0:
			logger.error(out.stderr)
			raise ErrorMessage('Something went wrong when creating the Dockerfile for repository "{}"'.format(repository_url))

			# Convert output to lines
			out_lines = out.stderr.split('\n')

			# Get rep2docker image name from output. Use "strip()" as sometimes the newline chars might jump in.
			last_line_message = json.loads(out_lines[-1])['message']
			if 'Reusing existing image' in last_line_message:
			repo2docker_image_name = last_line_message.split('(')[1].split(')')[0].strip()
			elif 'Successfully tagged' in last_line_message:
			repo2docker_image_name = last_line_message.split(' ')[2].strip()
			else:
			raise Exception('Cannot build')

			# Set image registry, name and tag. Use "strip()" as sometimes the newline chars might jump in.
			registry = os.environ.get('REGISTRY_HOST','proxy:5000').strip()
			image_name = repository_name.lower().strip()
			image_tag = repo2docker_image_name[-7:] # The last part of the image name generated by repo2docker is the git short hash

			# Re-tag image taking into account that if we are using the proxy as registry we use localhost or it won't work
			if registry == 'proxy:5000':
			push_registry = 'localhost:5000'
			else:
			push_registry = registry

			out = os_shell('sudo docker tag {} {}/{}:{}'.format(repo2docker_image_name,push_registry,image_name,image_tag) , capture=True)
			if out.exit_code != 0:
			logger.error(out.stderr)
			raise ErrorMessage('Something went wrong when tagging the container for repository "{}"'.format(repository_url))

			# Push image to the (local) registry
			out = os_shell('sudo docker push {}/{}:{}'.format(push_registry,image_name,image_tag) , capture=True)
			if out.exit_code != 0:
			logger.error(out.stderr)
			raise ErrorMessage('Something went wrong when pushing the container for repository "{}"'.format(repository_url))

			# Create the container if not already existent
			try:
			container = Container.objects.get(user=user, registry=registry, image_name=image_name, image_tag=image_tag)
			except Container.DoesNotExist:

			# Get name repo name and description from remote if we have and if we can
			repository_name_from_source= None
			repository_description_from_source = None
			if not container_name or not container_description:
			if repository_url.startswith('https://github.com'):
			try:
			response = requests.get('https://api.github.com/repos/{}'.format(repository_name))
			json_content = json.loads(response.content)
			repository_name_from_source = json_content['name'].title()
			repository_description_from_source = json_content['description']
			if not repository_description_from_source.endswith('.'):
			repository_description_from_source+='.'
			repository_description_from_source += ' Built from {}'.format(repository_url)
			except:
			pass

			# Set default container name and description
			if not container_name:
			container_name = repository_name_from_source if repository_name_from_source else repository_name

			if not container_description:
			container_description = repository_description_from_source if repository_description_from_source else 'Built from {}'.format(repository_url)

			# Ok, create the container
			container = Container.objects.create(user = user,
			name = container_name,
			description = container_description,
			registry = registry,
			image_name = image_name,
			image_tag = image_tag,
			image_arch = 'amd64',
			image_os = 'linux',
			interface_port = '8888',
			interface_protocol = 'http',
			interface_transport = 'tcp/ip',
			supports_custom_interface_port = False,
			supports_interface_auth = False)
			return container

services/webapp/code/rosetta/core_app/views.py

+97 −170

Original line number	Original line	Diff line number	Diff line
	@@ -12,7 +12,9 @@ from django.contrib.auth.models import User
	from django.shortcuts import redirect		from django.shortcuts import redirect
	from django.db.models import Q		from django.db.models import Q
	from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, Page		from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, Page
	from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_task_tunnel_host, get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation, sanitize_container_env_vars		from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_task_tunnel_host
			from .utils import get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation
			from .utils import sanitize_container_env_vars, get_or_create_container_from_repository
	from .decorators import public_view, private_view		from .decorators import public_view, private_view
	from .exceptions import ErrorMessage		from .exceptions import ErrorMessage

	@@ -899,11 +901,19 @@ def add_software(request):
	data = {}		data = {}
	data['user'] = request.user		data['user'] = request.user

			# Loop back the new container mode in the page to handle the switch
			data['new_container_from'] = request.GET.get('new_container_from', 'registry')

	# Container name if setting up a new container		# Container name if setting up a new container
	container_name = request.POST.get('container_name', None)		container_name = request.POST.get('container_name', None)

	if container_name:		if container_name:

			# How do we have to add this new container?
			new_container_from = request.POST.get('new_container_from', None)

			if new_container_from == 'registry':

	# Container description		# Container description
	container_description = request.POST.get('container_description', None)		container_description = request.POST.get('container_description', None)

	@@ -981,6 +991,18 @@ def add_software(request):
	supports_custom_interface_port = container_supports_custom_interface_port,		supports_custom_interface_port = container_supports_custom_interface_port,
	supports_interface_auth = container_supports_pass_auth,		supports_interface_auth = container_supports_pass_auth,
	env_vars = container_env_vars)		env_vars = container_env_vars)

			elif new_container_from == 'repository':

			container_description = request.POST.get('container_description', None)
			repository_url = request.POST.get('repository_url', None)
			repository_tag = request.POST.get('repository_tag', None)

			# The return type here is a container, not created
			get_or_create_container_from_repository(request.user, repository_url, repository_tag=repository_tag, container_name=container_name, container_description=container_description)

			else:
			raise Exception('Unknown new container mode "{}"'.format(new_container_from))
	# Set added switch		# Set added switch
	data['added'] = True		data['added'] = True

	@@ -1167,101 +1189,6 @@ def sharable_link_handler(request, short_uuid):
	return redirect(redirect_string)		return redirect(redirect_string)


	def get_or_create_container_from_repository(repository_url, repository_tag=None):
	repository_name = '{}/{}'.format(repository_url.split('/')[-2],repository_url.split('/')[-1])

	logger.debug('Called get_or_create_container_from_repository with repository_url="{}" and repository_tag="{}"'.format(repository_url,repository_tag))

	# If building:
	#{"message": "Successfully built 5a2089b2c334\n", "phase": "building"}
	#{"message": "Successfully tagged r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3:latest\n", "phase": "building"}

	# If reusing:
	#{"message": "Reusing existing image (r2dhttps-3a-2f-2fgithub-2ecom-2fnorvig-2fpytudes5e745c3), not building."}

	# Build the Docker container for this repo
	if repository_tag:
	command = 'sudo jupyter-repo2docker --ref {} --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_tag, repository_url)
	else:
	command = 'sudo jupyter-repo2docker --user-id 1000 --user-name rosetta --no-run --json-logs {}'.format(repository_url)
	out = os_shell(command, capture=True)
	if out.exit_code != 0:
	logger.error(out.stderr)
	raise ErrorMessage('Something went wrong when creating the Dockerfile for repository "{}"'.format(repository_url))

	# Convert output to lines
	out_lines = out.stderr.split('\n')

	# Get rep2docker image name from output
	last_line_message = json.loads(out_lines[-1])['message']
	if 'Reusing existing image' in last_line_message:
	repo2docker_image_name = last_line_message.split('(')[1].split(')')[0]
	elif 'Successfully tagged' in last_line_message:
	repo2docker_image_name = last_line_message.split(' ')[2]
	else:
	raise Exception('Cannot build')

	# Set image registry, name and tag, Use "strip()" as sometimes the newline chars might jump in.
	registry = os.environ.get('REGISTRY_HOST','proxy:5000').strip()
	image_name = repository_name.lower().strip()
	image_tag = repo2docker_image_name[-7:].strip() # The last part of the image name generated by repo2docker is the git short hash

	# Re-tag image taking into account that if we are using the proxy as registry we use localhost or it won't work
	if registry == 'proxy:5000':
	push_registry = 'localhost:5000'
	else:
	push_registry = registry

	out = os_shell('sudo docker tag {} {}/{}:{}'.format(repo2docker_image_name,push_registry,image_name,image_tag) , capture=True)
	if out.exit_code != 0:
	logger.error(out.stderr)
	raise ErrorMessage('Something went wrong when tagging the container for repository "{}"'.format(repository_url))

	# Push image to the (local) registry
	out = os_shell('sudo docker push {}/{}:{}'.format(push_registry,image_name,image_tag) , capture=True)
	if out.exit_code != 0:
	logger.error(out.stderr)
	raise ErrorMessage('Something went wrong when pushing the container for repository "{}"'.format(repository_url))

	# Create the container if not already existent
	try:
	container = Container.objects.get(registry=registry, image_name=image_name, image_tag=image_tag)
	except Container.DoesNotExist:

	# Set default container name and description
	container_name = repository_name
	container_description = 'Built from {}'.format(repository_url)

	# Get name repo name and description from GitHub (if repo is there)
	if repository_url.startswith('https://github.com'):
	try:
	response = requests.get('https://api.github.com/repos/{}'.format(repository_name))
	json_content = json.loads(response.content)
	container_name = json_content['name'].title()
	container_description = json_content['description']
	if not container_description.endswith('.'):
	container_description+='.'
	container_description += ' Built from {}'.format(repository_url)
	except:
	pass

	container = Container.objects.create(user = None,
	name = container_name,
	description = container_description,
	registry = registry,
	image_name = image_name,
	image_tag = image_tag,
	image_arch = 'amd64',
	image_os = 'linux',
	interface_port = '8888',
	interface_protocol = 'http',
	interface_transport = 'tcp/ip',
	supports_custom_interface_port = False,
	supports_interface_auth = False)
	return container



	#=========================		#=========================
	# New Binder Task		# New Binder Task
	#=========================		#=========================
	@@ -1280,7 +1207,7 @@ def new_binder_task(request, repository):
	repository_tag = repository.split('/')[-1]		repository_tag = repository.split('/')[-1]
	repository_url = repository.replace('/'+repository_tag, '')		repository_url = repository.replace('/'+repository_tag, '')

	container = get_or_create_container_from_repository(repository_url, repository_tag)		container = get_or_create_container_from_repository(request.user, repository_url, repository_tag)

	# Set the container		# Set the container
	data['task_container'] = container		data['task_container'] = container