Commit 226e7bc1 authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Added some tolerance for tunnel creation and a simple retry logic in the task connect view.

parent 7b06070a

services/webapp/code/rosetta/core_app/views.py

+20 −3
Original line number Original line Diff line number Diff line
@@ -6,6 +6,7 @@ import requests 
import socket

import socket

import subprocess

import subprocess

import base64

import base64

import time

import datetime

import datetime

from django.conf import settings

from django.conf import settings

from django.shortcuts import render

from django.shortcuts import render
@@ -1151,11 +1152,27 @@ def task_connect(request): 
        if task.container.interface_protocol.startswith('http'):

        if task.container.interface_protocol.startswith('http'):

            try:

            try:

                if task.requires_tcp_tunnel:

                if task.requires_tcp_tunnel:

                    # Check three times, as there might be some delay in establishing the tunnel

                    # in background in the above setup_tunnel_and_proxy() call

                    attempts = 0

                    while True:

                        try:

                            requests.get('{}://localhost:{}'.format(task.container.interface_protocol, task.tcp_tunnel_port), timeout=3)

                            requests.get('{}://localhost:{}'.format(task.container.interface_protocol, task.tcp_tunnel_port), timeout=3)

                else:

                    requests.get('{}://{}:{}'.format(task.container.interface_protocol, task.interface_ip, task.interface_port), timeout=3)

                            logger.debug('Task interface is answering')

                            logger.debug('Task interface is answering')

                            task.interface_status = 'running'

                            task.interface_status = 'running'

                        except:

                            if attempts > 2:

                                logger.debug('Too many attempts, giving up')

                                raise

                            else:

                                sleep_time = attempts + 1

                                logger.debug('Task interface not answering, retrying ({}s)...'.format(sleep_time))

                                attempts += 1

                                time.sleep(sleep_time)

                        else:

                            break

                else:

                    requests.get('{}://{}:{}'.format(task.container.interface_protocol, task.interface_ip, task.interface_port), timeout=3)

            except Exception as e:

            except Exception as e:

                logger.debug('Could not connect to task interface ({})'.format(e))

                logger.debug('Could not connect to task interface ({})'.format(e))