Fixed bug in creating tunnels and moved from "volumes" and "bindings" to "binds". (c450b440) · Commits · ExaCT / Rosetta

services/webapp/code/rosetta/core_app/computing_managers.py

+37 −37

Original line number	Diff line number	Diff line
		@@ -188,27 +188,27 @@ class RemoteComputingManager(ComputingManager):
		else:
		authstring = ''

		# Set bindings, only from sys config if the resource is not owned by the user
		# Set binds, only from sys config if the resource is not owned by the user
		if task.computing.user != task.user:
		bindings = task.computing.get_conf_param('bindings', from_sys_only=True )
		binds = task.computing.get_conf_param('binds', from_sys_only=True )
		else:
		bindings = task.computing.get_conf_param('bindings')
		if not bindings:
		bindings = ''
		binds = task.computing.get_conf_param('binds')
		if not binds:
		binds = ''
		else:
		bindings = '-B {}'.format(bindings)
		binds = '-B {}'.format(binds)

		# Manage task extra volumes
		if task.extra_volumes:
		if not bindings:
		bindings = '-B {}'.format(task.extra_volumes)
		# Manage task extra binds
		if task.extra_binds:
		if not binds:
		binds = '-B {}'.format(task.extra_binds)
		else:
		bindings += ',{}'.format(task.extra_volumes)
		binds += ',{}'.format(task.extra_binds)

		run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
		run_command += '/bin/bash -c \'"wget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> /dev/null && export BASE_PORT=\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid)
		run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)

		# Set registry
		if task.container.registry == 'docker_local':
		@@ -345,27 +345,27 @@ class SlurmComputingManager(ComputingManager):
		else:
		authstring = ''

		# Set bindings, only from sys config if the resource is not owned by the user
		# Set binds, only from sys config if the resource is not owned by the user
		if task.computing.user != task.user:
		bindings = task.computing.get_conf_param('bindings', from_sys_only=True )
		binds = task.computing.get_conf_param('binds', from_sys_only=True )
		else:
		bindings = task.computing.get_conf_param('bindings')
		if not bindings:
		bindings = ''
		binds = task.computing.get_conf_param('binds')
		if not binds:
		binds = ''
		else:
		bindings = '-B {}'.format(bindings)
		binds = '-B {}'.format(binds)

		# Manage task extra volumes
		if task.extra_volumes:
		if not bindings:
		bindings = '-B {}'.format(task.extra_volumes)
		# Manage task extra binds
		if task.extra_binds:
		if not binds:
		binds = '-B {}'.format(task.extra_binds)
		else:
		bindings += ',{}'.format(task.extra_volumes)
		binds += ',{}'.format(task.extra_binds)

		run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, user, host)
		run_command += '\'bash -c "echo \\"#!/bin/bash\nwget {}/api/v1/base/agent/?task_uuid={} -O \$HOME/agent_{}.py &> \$HOME/{}.log && export BASE_PORT=\\\\\\$(python \$HOME/agent_{}.py 2> \$HOME/{}.log) && '.format(webapp_conn_string, task.uuid, task.uuid, task.uuid, task.uuid, task.uuid)
		run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\\\\\\$BASE_PORT && {} '.format(authstring)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)

		# Double to escape for Pythom, six for shell (double times three as \\\ escapes a single slash in shell)

		@@ -498,22 +498,22 @@ class RemotehopComputingManager(ComputingManager):
		else:
		authstring = ''

		# Set bindings, only from sys config if the resource is not owned by the user
		# Set binds, only from sys config if the resource is not owned by the user
		if task.computing.user != task.user:
		bindings = task.computing.get_conf_param('bindings', from_sys_only=True )
		binds = task.computing.get_conf_param('binds', from_sys_only=True )
		else:
		bindings = task.computing.get_conf_param('bindings')
		if not bindings:
		bindings = ''
		binds = task.computing.get_conf_param('binds')
		if not binds:
		binds = ''
		else:
		bindings = '-B {}'.format(bindings)
		binds = '-B {}'.format(binds)

		# Manage task extra volumes
		if task.extra_volumes:
		if not bindings:
		bindings = '-B {}'.format(task.extra_volumes)
		# Manage task extra binds
		if task.extra_binds:
		if not binds:
		binds = '-B {}'.format(task.extra_binds)
		else:
		bindings += ',{}'.format(task.extra_volumes)
		binds += ',{}'.format(task.extra_binds)

		run_command = 'ssh -i {} -4 -o StrictHostKeyChecking=no {}@{} '.format(user_keys.private_key_file, first_user, first_host)
		run_command += '"ssh -4 -o StrictHostKeyChecking=no {}@{} /bin/bash -c \''.format(second_user, second_host)
		@@ -523,13 +523,13 @@ class RemotehopComputingManager(ComputingManager):
		if setup_command:
		run_command += setup_command + ' && '
		run_command += '\'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT=\$BASE_PORT && {} '.format(authstring)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)
		else:
		run_command += ' : && ' # Trick to prevent some issues in exporting variables
		if setup_command:
		run_command += setup_command + ' && '
		run_command += 'export SINGULARITY_NOHTTPS=true && export SINGULARITYENV_BASE_PORT={} && {} '.format(task.port, authstring)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(bindings)
		run_command += 'exec nohup singularity run {} --pid --writable-tmpfs --containall --cleanenv '.format(binds)

		# Set registry
		if task.container.registry == 'docker_local':

services/webapp/code/rosetta/core_app/models.py

+1 −1

Original line number	Diff line number	Diff line
		@@ -272,7 +272,7 @@ class Task(models.Model):
		port = models.IntegerField('Task port', blank=True, null=True)
		ip = models.CharField('Task ip address', max_length=36, blank=True, null=True)
		tunnel_port = models.IntegerField('Task tunnel port', blank=True, null=True)
		extra_volumes = models.CharField('Extra volumes', max_length=4096, blank=True, null=True)
		extra_binds = models.CharField('Extra binds', max_length=4096, blank=True, null=True)

		# Links
		computing = models.ForeignKey(Computing, related_name='+', on_delete=models.CASCADE)

services/webapp/code/rosetta/core_app/templates/components/task.html

+2 −2

Original line number	Diff line number	Diff line
		@@ -47,8 +47,8 @@
		</tr>

		<tr>
		<td><b>Extra volumes</b></td>
		<td>{{ task.extra_volumes }}</td>
		<td><b>Extra binds</b></td>
		<td>{{ task.extra_binds }}</td>
		</tr>

		<tr>

services/webapp/code/rosetta/core_app/templates/create_task.html

+3 −3

Original line number	Diff line number	Diff line
		@@ -153,10 +153,10 @@

		{% if data.task_container.type == 'singularity' %}
		<tr>
		<td valign="top" style="width:180px"><b>Extra volumes</b></td>
		<td valign="top" style="width:180px"><b>Extra binds</b></td>
		<td>
		<input type="text" name="extra_volumes" value="" placeholder="" size="23" /><br>
		<p style="line-height: 0.95"><font size=-1>Here you can set extra volume bindings on top of the ones already define by the administrator. Format is <i>host_directory:container_directory</i>, comma separated.</font></p>
		<p style="line-height: 0.95"><font size=-1>Here you can set extra binds on top of the ones already define by the administrator. Format is <i>host_directory:container_directory</i>, comma separated.</font></p>
		</td>
		</tr>
		{% endif %}
		@@ -195,7 +195,7 @@

		<tr><td colspan=2>
		<table><tr><td style="border: 1px solid lightgray;" >
		I understand that files saved or modified in this container, if not explicitly saved to a persistent volume, will be LOST when the task ends.
		I understand that files saved or modified in this container, if not explicitly saved to a persistent bind, will be LOST when the task ends.
		</td><td style="border: 1px solid lightgray;" >
		<input class="form-check-input" type="checkbox" value="" id="invalidCheck" required>
		</td></table>

services/webapp/code/rosetta/core_app/views.py

+13 −7

Original line number	Diff line number	Diff line
		@@ -392,6 +392,9 @@ def tasks(request):
		else:
		logger.debug('Task "{}" has no running tunnel, creating it'.format(task))

		# Get user keys
		user_keys = KeyPair.objects.get(user=task.user, default=True)

		# Tunnel command
		if task.computing.type == 'remotehop':

		@@ -403,11 +406,14 @@ def tasks(request):
		#setup_command = task.computing.get_conf_param('setup_command')
		#base_port = task.computing.get_conf_param('base_port')

		tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(task.tunnel_port, task.ip, task.port, first_user, first_host)
		tunnel_command= 'ssh -4 i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} {}@{} & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port, first_user, first_host)

		else:
		elif task.computing.type == 'local':
		tunnel_command= 'ssh -4 -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(task.tunnel_port, task.ip, task.port)

		else:
		tunnel_command= 'ssh -4 i {} -o StrictHostKeyChecking=no -nNT -L 0.0.0.0:{}:{}:{} localhost & '.format(user_keys.private_key_file, task.tunnel_port, task.ip, task.port)

		background_tunnel_command = 'nohup {} >/dev/null 2>&1 &'.format(tunnel_command)

		# Log
		@@ -561,7 +567,7 @@ def create_task(request):
		computing_cpus = request.POST.get('computing_cpus', None)
		computing_memory = request.POST.get('computing_memory', None)
		computing_partition = request.POST.get('computing_partition', None)
		extra_volumes = request.POST.get('extra_volumes', None)
		extra_binds = request.POST.get('extra_binds', None)

		computing_options = {}
		if computing_cpus:
		@@ -588,8 +594,8 @@ def create_task(request):
		if task.container.ports:
		task.port = task.container.port

		# Set exttra volumes if any:
		task.extra_volumes = extra_volumes
		# Set extra binds if any:
		task.extra_binds = extra_binds

		# Save the task before starting it, or the computing manager will not be able to work properly
		task.save()