.gitignore

@@ -16,7 +16,6 @@ data*
 
 # DB conf
 services/webapp/db_conf.sh
-services/proxy/certificates
 
 # Compose
 docker-compose.yml

README.md

@@ -50,8 +50,9 @@ Webapp service configuraion parameters and their defaults:
       - DJANGO_DEBUG=true
       - DJANGO_LOG_LEVEL=ERROR
       - ROSETTA_LOG_LEVEL=ERROR
-      - ROSETTA_HOST=localhost
-      - ROSETTA_TUNNEL_HOST=localhost
+      - ROSETTA_HOST=localhost      
+      - ROSETTA_TASKS_PROXY_HOST=$ROSETTA_HOST
+      - ROSETTA_TASKS_TUNNEL_HOST=$ROSETTA_HOST  
       - ROSETTA_WEBAPP_HOST=""
       - ROSETTA_WEBAPP_PORT=8080
       - ROSETTA_REGISTRY_HOST=proxy
@@ -69,11 +70,33 @@ Webapp service configuraion parameters and their defaults:
 
 Notes:
 
- - `ROSETTA_TUNNEL_HOST` must not include http:// or https://
  - `ROSETTA_REGISTRY_HOST` should be set to the same value as `ROSETTA_HOST` for production scenarios, in order to be secured unders SSL. The `standaloneworker` is configured to treat the following hosts (and ports) as unsecure registies, where it can connect without a valid certificate: `proxy:5000`,`dregistry:5000` and `rosetta.platform:5000`.
  - `ROSETTA_WEBAPP_HOST` is used for let the agent know where to connect, and it is differentiated from `ROSETTA_HOST` as it can be on an internal Docker network. It is indeed defaulted to the `webapp` container IP address.
 
+Proxy service configuraion parameters and their defaults:
+
+      - SAFEMODE=false
+      - ROSETTA_HOST=localhost
+      - ROSETTA_TASKS_PROXY_HOST=$ROSETTA_HOST
+
+
+### Certificates for the proxy
+
+Certificates can be automatically handled with Letsencrypt. By default, a snakeoil certificate is used. To set up letsencrypt, first of all run inside the proxy (only once in its lifetime):
 
+	$ sudo rm -rf /etc/letsencrypt/live/YOUR_ROSETTA_HOST (or ROSETTA_TASKS_PROXY_HOST)
+
+Then, edit the `/etc/apache2/sites-available/proxy-global.conf` file and change the certificates for the domain that you want to enable with Letsencrypt to use snakeoils (otherwise nex comamnd will fail), then:
+
+	$  sudo apache2ctl -k graceful
+
+Now:
+
+    $ sudo certbot certonly --apache --register-unsafely-without-email --agree-tos -d YOUR_ROSETTA_HOST (or ROSETTA_TASKS_PROXY_HOST)
+    
+...or for the domain that you want to enable with Letsencrypt. This will initialize the certificate in /etc/letsencypt, which is stored on the host in `./data/proxy/letsencrypt`
+
+Finally, re-change the `/etc/apache2/sites-available/proxy-global.conf` file to use the correct certificates for the domain (or just restart the proxy service but wiht clean and then run).
 
 ### User types 
 In Rosetta there are two user types: standard users and power users. Their type is set in their user profile, and only power users can:
@@ -120,17 +143,36 @@ Note that when you edit the Django ORM model, you need to make migrations and ap
 
 
     
-### Logs and testing
+### Testing
 
 Run Web App unit tests (with Rosetta running)
+    
+    $ rosetta/test
+
+
+### Logs
+
+
+Chek out logs for Docker containers (including entrypoints):
+
+
+    $ rosetta/logs web
 
-    $ rosetta/logs webapp
+    $ rosetta/logs proxy
+
+
+Chek out logs for supervisord services:
+
+        
+    $ rosetta/logs web startup
     
-    $ rosetta/logs webapp startup
+    $ rosetta/logs web server
+
+    $ rosetta/logs proxy apache
+    
+    $ rosetta/logs proxy certbot
     
-    $ rosetta/logs webapp server
     
-    $ rosetta/test
     
     
 ### Computing resources requirements

docker-compose-dev.yml

@@ -60,15 +60,14 @@ services:
       - ROSETTA_LOG_LEVEL=DEBUG
       #- ROSETTA_WEBAPP_HOST=localhost # Internal, for the agent
       #- ROSETTA_WEBAPP_PORT=8080      # Internal, for the agent
-      #- ROSETTA_REGISTRY_HOST=
+      #- ROSETTA_REGISTRY_HOST=proxy
       #- ROSETTA_REGISTRY_PORT=5000
       #- DJANGO_EMAIL_APIKEY=""
       #- DJANGO_EMAIL_FROM="Rosetta Platform <notifications@rosetta.platform>"
       #- DJANGO_SECRET_KEY=""
-      - TASK_PROXY_HOST=localhost
-      - TASK_TUNNEL_HOST=localhost
+      #- ROSETTA_TASKS_PROXY_HOST=
+      #- ROSETTA_TASKS_TUNNEL_HOST=
       - ROSETTA_HOST=localhost
-      - REGISTRY_HOST=proxy:5000     # Use same value as ROSETTA_HOST for production or to use "real" computing resurces 
     ports:
       - "8080:8080"
       - "7000-7020:7000-7020"
@@ -86,14 +85,16 @@ services:
     environment:
       - SAFEMODE=False
       - ROSETTA_HOST=localhost
+      - ROSETTA_TASKS_PROXY_HOST=localhost
     ports:
       - "80:80"
       - "443:443"
       - "9000-9020:9000-9020"
       - "5000:5000"
     volumes:
-      - ./data/shared:/shared   
-
+      - ./data/shared:/shared
+      - ./data/proxy/letsencrypt:/etc/letsencrypt
+      - ./data/proxy/log:/var/log/proxy
 
 
 

rosetta/clean

@@ -9,5 +9,5 @@ fi
 if [[ $# -eq 0 ]] ; then
     docker-compose down
 else
-    docker-compose down $@
+    docker-compose rm -s -v -f $@
 fi

rosetta/setup

@@ -9,15 +9,6 @@ else
 fi
 
 
-# Use dev certificates if not already set up
-if [ ! -d services/proxy/certificates ]; then
-    echo "Using dev certificates."
-    cp -a services/proxy/certificates-dev  services/proxy/certificates
-else
-    echo "Not using dev certificates as certificates are already present."
-fi
-
-
 # Use dev docker-compose.yml if not already set up
 if [ ! -f docker-compose.yml ]; then
     echo "Using dev docker-compose.yml"

services/proxy/Dockerfile

@@ -10,10 +10,17 @@ RUN apt-get update
 RUN apt-get install -y apache2
 RUN apt-get install apache2-utils
 
-# Copy conf
+# Install Certbot
+RUN apt-get install certbot python3-certbot-apache -y
+
+# Supervisord scripts
 COPY supervisord_apache.conf /etc/supervisor/conf.d/
-COPY run_Apache.sh /etc/supervisor/conf.d/
-RUN chmod 755 /etc/supervisor/conf.d/run_Apache.sh
+COPY run_apache.sh /etc/supervisor/conf.d/
+RUN chmod 755 /etc/supervisor/conf.d/run_apache.sh
+
+COPY supervisord_certbot.conf /etc/supervisor/conf.d/
+COPY run_certbot.sh /etc/supervisor/conf.d/
+RUN chmod 755 /etc/supervisor/conf.d/run_certbot.sh
 
 # Enable mod_proxy and SSL
 RUN a2enmod proxy
@@ -22,21 +29,21 @@ RUN sudo a2enmod ssl
 RUN a2enmod rewrite
 RUN a2enmod headers
 RUN a2enmod proxy_wstunnel
- 
+
 # Clean up default stuff
 RUN rm /etc/apache2/sites-available/000-default.conf
 RUN rm /etc/apache2/sites-enabled/000-default.conf
 RUN rm /etc/apache2/sites-available/default-ssl.conf
 #RUN rm /etc/apache2/sites-enabled/default-ssl.conf
 
-# Copy certificates (snakeoil or real)
-RUN mkdir /certificates
-COPY certificates/rosetta_platform.crt /root/certificates/rosetta_platform/rosetta_platform.crt
-COPY certificates/rosetta_platform.key /root/certificates/rosetta_platform/rosetta_platform.key
-COPY certificates/rosetta_platform.ca-bundle /root/certificates/rosetta_platform/rosetta_platform.ca-bundle
-COPY certificates/rosetta_tasks.crt /root/certificates/rosetta_platform/rosetta_tasks.crt
-COPY certificates/rosetta_tasks.key /root/certificates/rosetta_platform/rosetta_tasks.key
-COPY certificates/rosetta_tasks.ca-bundle /root/certificates/rosetta_platform/rosetta_tasks.ca-bundle
+# Apache conf
+COPY apache2.conf /etc/apache2/apache2.conf 
+
+# Copy self-signed (snakeoil) certificates
+RUN mkdir /root/certificates
+COPY certificates/selfsigned.crt /root/certificates/selfsigned.crt
+COPY certificates/selfsigned.key /root/certificates/selfsigned.key
+COPY certificates/selfsigned.ca-bundle /root/certificates/selfsigned.ca-bundle
 
 # Copy index and norobots.txt
 COPY index.html /var/www/html/

services/proxy/apache2.conf

+# This is the main Apache server configuration file.  It contains the
+# configuration directives that give the server its instructions.
+# See http://httpd.apache.org/docs/2.4/ for detailed information about
+# the directives and /usr/share/doc/apache2/README.Debian about Debian specific
+# hints.
+#
+#
+# Summary of how the Apache 2 configuration works in Debian:
+# The Apache 2 web server configuration in Debian is quite different to
+# upstream's suggested way to configure the web server. This is because Debian's
+# default Apache2 installation attempts to make adding and removing modules,
+# virtual hosts, and extra configuration directives as flexible as possible, in
+# order to make automating the changes and administering the server as easy as
+# possible.
+
+# It is split into several files forming the configuration hierarchy outlined
+# below, all located in the /etc/apache2/ directory:
+#
+#   /etc/apache2/
+#   |-- apache2.conf
+#   |   `--  ports.conf
+#   |-- mods-enabled
+#   |   |-- *.load
+#   |   `-- *.conf
+#   |-- conf-enabled
+#   |   `-- *.conf
+#   `-- sites-enabled
+#       `-- *.conf
+#
+#
+# * apache2.conf is the main configuration file (this file). It puts the pieces
+#   together by including all remaining configuration files when starting up the
+#   web server.
+#
+# * ports.conf is always included from the main configuration file. It is
+#   supposed to determine listening ports for incoming connections which can be
+#   customized anytime.
+#
+# * Configuration files in the mods-enabled/, conf-enabled/ and sites-enabled/
+#   directories contain particular configuration snippets which manage modules,
+#   global configuration fragments, or virtual host configurations,
+#   respectively.
+#
+#   They are activated by symlinking available configuration files from their
+#   respective *-available/ counterparts. These should be managed by using our
+#   helpers a2enmod/a2dismod, a2ensite/a2dissite and a2enconf/a2disconf. See
+#   their respective man pages for detailed information.
+#
+# * The binary is called apache2. Due to the use of environment variables, in
+#   the default configuration, apache2 needs to be started/stopped with
+#   /etc/init.d/apache2 or apache2ctl. Calling /usr/bin/apache2 directly will not
+#   work with the default configuration.
+
+
+# Global configuration
+#
+
+#
+# ServerRoot: The top of the directory tree under which the server's
+# configuration, error, and log files are kept.
+#
+# NOTE!  If you intend to place this on an NFS (or otherwise network)
+# mounted filesystem then please read the Mutex documentation (available
+# at <URL:http://httpd.apache.org/docs/2.4/mod/core.html#mutex>);
+# you will save yourself a lot of trouble.
+#
+# Do NOT add a slash at the end of the directory path.
+#
+#ServerRoot "/etc/apache2"
+
+#
+# The accept serialization lock file MUST BE STORED ON A LOCAL DISK.
+#
+#Mutex file:${APACHE_LOCK_DIR} default
+
+#
+# The directory where shm and other runtime files will be stored.
+#
+
+DefaultRuntimeDir ${APACHE_RUN_DIR}
+
+#
+# PidFile: The file in which the server should record its process
+# identification number when it starts.
+# This needs to be set in /etc/apache2/envvars
+#
+PidFile ${APACHE_PID_FILE}
+
+#
+# Timeout: The number of seconds before receives and sends time out.
+#
+Timeout 300
+
+#
+# KeepAlive: Whether or not to allow persistent connections (more than
+# one request per connection). Set to "Off" to deactivate.
+#
+KeepAlive On
+
+#
+# MaxKeepAliveRequests: The maximum number of requests to allow
+# during a persistent connection. Set to 0 to allow an unlimited amount.
+# We recommend you leave this number high, for maximum performance.
+#
+MaxKeepAliveRequests 100
+
+#
+# KeepAliveTimeout: Number of seconds to wait for the next request from the
+# same client on the same connection.
+#
+KeepAliveTimeout 5
+
+
+# These need to be set in /etc/apache2/envvars
+User ${APACHE_RUN_USER}
+Group ${APACHE_RUN_GROUP}
+
+#
+# HostnameLookups: Log the names of clients or just their IP addresses
+# e.g., www.apache.org (on) or 204.62.129.132 (off).
+# The default is off because it'd be overall better for the net if people
+# had to knowingly turn this feature on, since enabling it means that
+# each client request will result in AT LEAST one lookup request to the
+# nameserver.
+#
+HostnameLookups Off
+
+# ErrorLog: The location of the error log file.
+# If you do not specify an ErrorLog directive within a <VirtualHost>
+# container, error messages relating to that virtual host will be
+# logged here.  If you *do* define an error logfile for a <VirtualHost>
+# container, that host's errors will be logged there and not here.
+#
+#ErrorLog ${APACHE_LOG_DIR}/error.log
+ErrorLog /dev/stderr
+TransferLog /dev/stdout
+
+#
+# LogLevel: Control the severity of messages logged to the error_log.
+# Available values: trace8, ..., trace1, debug, info, notice, warn,
+# error, crit, alert, emerg.
+# It is also possible to configure the log level for particular modules, e.g.
+# "LogLevel info ssl:warn"
+#
+LogLevel warn
+
+# Include module configuration:
+IncludeOptional mods-enabled/*.load
+IncludeOptional mods-enabled/*.conf
+
+# Include list of ports to listen on
+Include ports.conf
+
+
+# Sets the default security model of the Apache2 HTTPD server. It does
+# not allow access to the root filesystem outside of /usr/share and /var/www.
+# The former is used by web applications packaged in Debian,
+# the latter may be used for local directories served by the web server. If
+# your system is serving content from a sub-directory in /srv you must allow
+# access here, or in any related virtual host.
+<Directory />
+    Options FollowSymLinks
+    AllowOverride None
+    Require all denied
+</Directory>
+
+<Directory /usr/share>
+    AllowOverride None
+    Require all granted
+</Directory>
+
+<Directory /var/www/>
+    Options Indexes FollowSymLinks
+    AllowOverride None
+    Require all granted
+</Directory>
+
+#<Directory /srv/>
+#   Options Indexes FollowSymLinks
+#   AllowOverride None
+#   Require all granted
+#</Directory>
+
+
+
+
+# AccessFileName: The name of the file to look for in each directory
+# for additional configuration directives.  See also the AllowOverride
+# directive.
+#
+AccessFileName .htaccess
+
+#
+# The following lines prevent .htaccess and .htpasswd files from being
+# viewed by Web clients.
+#
+<FilesMatch "^\.ht">
+    Require all denied
+</FilesMatch>
+
+
+#
+# The following directives define some format nicknames for use with
+# a CustomLog directive.
+#
+# These deviate from the Common Log Format definitions in that they use %O
+# (the actual bytes sent including headers) instead of %b (the size of the
+# requested file), because the latter makes it impossible to detect partial
+# requests.
+#
+# Note that the use of %{X-Forwarded-For}i instead of %h is not recommended.
+# Use mod_remoteip instead.
+#
+LogFormat "%v:%p %h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" vhost_combined
+LogFormat "%h %l %u %t \"%r\" %>s %O \"%{Referer}i\" \"%{User-Agent}i\"" combined
+LogFormat "%h %l %u %t \"%r\" %>s %O" common
+LogFormat "%{Referer}i -> %U" referer
+LogFormat "%{User-agent}i" agent
+
+# Include of directories ignores editors' and dpkg's backup files,
+# see README.Debian for details.
+
+# Include generic snippets of statements
+IncludeOptional conf-enabled/*.conf
+
+# Include the virtual host configurations:
+IncludeOptional sites-enabled/*.conf
+
+# vim: syntax=apache ts=4 sw=4 sts=4 sr noet
\ No newline at end of file

services/proxy/certificates/selfsigned.crt

+-----BEGIN CERTIFICATE-----
+MIICvjCCAaagAwIBAgIJAM9K5eCAXmjJMA0GCSqGSIb3DQEBCwUAMBcxFTATBgNV
+BAMTDDAyNWQ5YzY3ZThlZDAeFw0xNzEwMjMxODM2NTdaFw0yNzEwMjExODM2NTda
+MBcxFTATBgNVBAMTDDAyNWQ5YzY3ZThlZDCCASIwDQYJKoZIhvcNAQEBBQADggEP
+ADCCAQoCggEBAMe3pfnmJEpE8GZKRZ/XO034gRlTgmylpmWRQ59wYArujUmNRAB3
+43GsdQvxavponG2x97HKHtDfrPdLycwWxyDFKIQw0e6GY0H4gMBJq5UjA9O7sznF
+Sy1fjUIfhz2tIbBdo1BgW9EUzgq6sKBN8H6RnzSat+DTcP3xzUhkTFS6EPi8MFXw
+gvBlEUWGjjesaaylLfD5/eDPhEeciKN8mlVi4I84QkPgu/O4NHXcqT1LaZUUPojs
+Ca2bq87yyuSQljQby2c/4GqPue2PF3EU3gp051lWbOvTPkPuA0HC5VsDFZKxJHfL
+b5gyN+I1tKjkKe8BpTNmQ8DP5SDi/ut4ltcCAwEAAaMNMAswCQYDVR0TBAIwADAN
+BgkqhkiG9w0BAQsFAAOCAQEACf5KUw0rM9jgyJWa1g2mhGpmqY4aiaMwJrwYYZza
+DtYS2MQoiCyIpO3PHlPfLVHtvrU+5OX+YMPfA1wWRtSWu86Vh+JVU/56ZdEul0y5
+aH1s8aYDaiDJa2ee5MBnqMMNbkcoMLuKens4+uOTTKolJduZNfZI/yk2JQQvMdTi
+qYB7dg/V5fe/lba8OoUjN+m3J4KS/bqEOZ32/PLTHFwv1/z4osXXUEkzD2hQzBR2
+ru2XD/7+oeR729r7YcskHRdJd8LVERQOi0JK1qyr4gEB67JvaCiOFFc6qzwqntdx
+UwVRxLa5KbetgD1Fw/ic+9TNHmB9Y/9mBtpGx8KufWpoYg==
+-----END CERTIFICATE-----

services/proxy/certificates/selfsigned.key

+-----BEGIN PRIVATE KEY-----
+MIIEvgIBADANBgkqhkiG9w0BAQEFAASCBKgwggSkAgEAAoIBAQDHt6X55iRKRPBm
+SkWf1ztN+IEZU4JspaZlkUOfcGAK7o1JjUQAd+NxrHUL8Wr6aJxtsfexyh7Q36z3
+S8nMFscgxSiEMNHuhmNB+IDASauVIwPTu7M5xUstX41CH4c9rSGwXaNQYFvRFM4K
+urCgTfB+kZ80mrfg03D98c1IZExUuhD4vDBV8ILwZRFFho43rGmspS3w+f3gz4RH
+nIijfJpVYuCPOEJD4LvzuDR13Kk9S2mVFD6I7Amtm6vO8srkkJY0G8tnP+Bqj7nt
+jxdxFN4KdOdZVmzr0z5D7gNBwuVbAxWSsSR3y2+YMjfiNbSo5CnvAaUzZkPAz+Ug
+4v7reJbXAgMBAAECggEACG6hjFaCK7yTZc42+FOvBlC6qqYS+KFZ0Cn87+tfsrZ1
+sqhLObXWHYOJgZKU0LO//wWnjpMZD/qRo/NINtyzVZfdaQ9ina6A3FUwom2519cd
+nz/qhkLlNKo3HZaVMC5yIK8jaQ5YchBtzpgpQutnfwCI90CdCNoEiERARZEug9kw
+Km30UM3sh1fAvAQuynkaKKHzxF5j1pnNVDGWZSb3qBN7kTSaR1P6wBKYAJxPj1vP
+rL0PXLGfzkuryAXfgVq8hsYIyg6VwCmaontPItopmH5U9sYo0gB5RLDyv9++n3I6
+lkeSyCqQpGCA6ugiqsO8s4bL0thTi9YLL3ztg+sF8QKBgQDoc8W0ALSY+9YWV8r7
+K+BW0kVzatJmwUdLLEGw2l23ScjS6yQuNlxJCav2+n6bkM73VbM+lSjX1RaFsQty
+52EPefTf+7FADQnk7eanPXvlb1nDKb+vitl6MV5okJBz2jZRNPa7bcpFBUCejdGJ
+VZX32gdsxk+9VPjJQb2Y6tkoOQKBgQDb8vTvufcMiGC2q8Y7XrQ84ZSFHGQUHf4Z
+R909fPGprfWuGQG2/DFEX1R9lCIth+gioWGLrboU5Q7+u2cn81s0zPlZI6Kg2e2Q
+htVYaMSw731yWsilH7j6RftBbYUb/3jZadVT4FbDmxJqeiJVzQXlLJ3C9dExMj0n
+weZM2f4XjwKBgQC/CQZd3IaPg8h6LEShD3obYEu7gvrPf+B7oy+JjKygSX9F+AGQ
+CRTm4Y/2Nf9/Eg9FraTVtfgPCQytasch844NDgl1WoBdR1nuTqXUo+8Cq/R1NAZY
+2h/JEHGqNcTBsYAaVRDBEIW/G4XzyFGAMFpDi2e2uXQnAYJExEZxOfCl4QKBgDIH
+inU47JvaLX1/hwCcIw0yFnFMqur0g4bGlOlWkTWSTy7Bm2U+6gnuUS6bUkbfAgtW
+f/SgmJIGJCoHAIjSzu0sro77DxPdXi8grEiG1C6W2wb25WrB03aCEouoWL2sl5WE
+gDSq87FchYzYqRSxJOUjB+N/vIyfK8/uR+81Kpm7AoGBAJjZ0HGFuOn7FQfg1/gJ
+5jQz3/JvjemQO83xDuh/mczRWNvmeC3H0Few8MfcJuwORQRbZ1e5uTRG9DOOZKOk
+1/fEpJZvXdQxLtITwx6yq/BG857pA0C8rNCW+OzjfhSwbI4rt4COwjnZF8e9Lbi6
+0DANJf8JhgYxKPeUT2au+147
+-----END PRIVATE KEY-----

services/proxy/prestartup_proxy.sh

-# Nothing here..
+#!/bin/bash
+
+#-----------------------
+# Rosetta Platform
+#-----------------------
+
+# Always create dir if not existent
+mkdir -p /etc/letsencrypt/live/$ROSETTA_HOST/
+
+# If there are no certificates, use snakeoils
+if [ ! -f "/etc/letsencrypt/live/$ROSETTA_HOST/cert.pem" ]; then
+    echo "Using default self-signed certificate cer file for $ROSETTA_HOST as not existent..."
+    cp -a /root/certificates/selfsigned.crt /etc/letsencrypt/live/$ROSETTA_HOST/cert.pem
+else
+    echo "Not using default self-signed certificate cer file for $ROSETTA_HOST as already existent."
+fi
+
+if [ ! -f "/etc/letsencrypt/live/$ROSETTA_HOST/privkey.pem" ]; then
+    echo "Using default self-signed certificate privkey file for $ROSETTA_HOST as not existent..."
+    cp -a /root/certificates/selfsigned.key /etc/letsencrypt/live/$ROSETTA_HOST/privkey.pem
+else
+    echo "Not using default self-signed certificate privkey file for $ROSETTA_HOST as already existent."
+fi
+
+if [ ! -f "/etc/letsencrypt/live/$ROSETTA_HOST/fullchain.pem" ]; then
+    echo "Using default self-signed certificate fullchain file for $ROSETTA_HOST as not existent..."
+    cp -a /root/certificates/selfsigned.ca-bundle /etc/letsencrypt/live/$ROSETTA_HOST/fullchain.pem
+else
+    echo "Not using default self-signed certificate fullchain file for $ROSETTA_HOST as already existent."
+fi
+
+# Replace the ROSETTA_HOST in the Apache proxy conf. Directly using an env var doen not wotk
+# with the letsencryot client, which has a bug: https://github.com/certbot/certbot/issues/8243
+sudo sed -i "s/__ROSETTA_HOST__/$ROSETTA_HOST/g" /etc/apache2/sites-available/proxy-global.conf
+    
+
+#-----------------------
+# Rosetta tasks
+#-----------------------
+
+# If the tasks host is equal to rosetta host or not set, do nothing as we have already habdled it above
+if [ "x$ROSETTA_TASKS_PROXY_HOST" == "x$ROSETTA_HOST" ] || [ "x$ROSETTA_TASKS_PROXY_HOST" == "x" ]; then
+    echo "[INFO] Not setting up certificates forRosetta tasks host as qual to Rosetta main host"
+    ROSETTA_TASKS_PROXY_HOST=$ROSETTA_HOST
+else
+
+    # Always create dir if not existent
+    mkdir -p /etc/letsencrypt/live/$ROSETTA_TASKS_PROXY_HOST/
+
+    # If there are no certificates, use snakeoils
+	if [ ! -f "/etc/letsencrypt/live/$ROSETTA_TASKS_PROXY_HOST/cert.pem" ]; then
+	    echo "Using default self-signed certificate cer file for $ROSETTA_TASKS_PROXY_HOST as not existent..."
+	    cp -a /root/certificates/selfsigned.crt /etc/letsencrypt/live/$ROSETTA_TASKS_PROXY_HOST/cert.pem
+	else
+	    echo "Not using default self-signed certificate cer file for $ROSETTA_TASKS_PROXY_HOST as already existent."
+	fi
+	
+	if [ ! -f "/etc/letsencrypt/live/$ROSETTA_TASKS_PROXY_HOST/privkey.pem" ]; then
+	    echo "Using default self-signed certificate privkey file for $ROSETTA_TASKS_PROXY_HOST as not existent..."
+	    cp -a /root/certificates/selfsigned.key /etc/letsencrypt/live/$ROSETTA_TASKS_PROXY_HOST/privkey.pem
+	else
+	    echo "Not using default self-signed certificate privkey file for $ROSETTA_TASKS_PROXY_HOST as already existent."
+	fi
+	
+	if [ ! -f "/etc/letsencrypt/live/$ROSETTA_TASKS_PROXY_HOST/fullchain.pem" ]; then
+	    echo "Using default self-signed certificate fullchain file for $ROSETTA_TASKS_PROXY_HOST as not existent..."
+	    cp -a /root/certificates/selfsigned.ca-bundle /etc/letsencrypt/live/$ROSETTA_TASKS_PROXY_HOST/fullchain.pem
+	else
+	    echo "Not using default self-signed certificate fullchain file for $ROSETTA_TASKS_PROXY_HOST as already existent."
+	fi
+
+fi
+
+# Replace the __ROSETTA_TASKS_PROXY_HOST__ in the Apache proxy conf. Directly using an env var doen not wotk
+# with the letsencryot client, which has a bug: https://github.com/certbot/certbot/issues/8243
+sudo sed -i "s/__ROSETTA_TASKS_PROXY_HOST__/$ROSETTA_TASKS_PROXY_HOST/g" /etc/apache2/sites-available/proxy-global.conf
+    

services/proxy/proxy-global.conf

-<VirtualHost *:80>
 
-    ServerAdmin admin@rosetta.platform
+# Note: if setting up Let's Encrypt, use these certificates for the 
+# intermediate step, according to the REANDE:
+# SSLCertificateFile /root/certificates/selfsigned.crt
+# SSLCertificateKeyFile /root/certificates/selfsigned.key
+# SSLCACertificateFile /root/certificates/selfsigned.ca-bundle
 
-    #----------------------------------
-    # Force https (except on localhost)
-    #----------------------------------
 
+#------------------------
+# Force https
+#------------------------
+<VirtualHost *:80>
+
+    ServerAdmin admin@rosetta.platform
     RewriteEngine On
     RewriteCond %{HTTPS} off
-    RewriteCond %{HTTP_HOST} !=localhost
     RewriteRule (.*) https://%{HTTP_HOST}%{REQUEST_URI}
-    
-    ProxyPass / http://webapp:8080/
-    ProxyPassReverse / http://webapp:8080/ 
     AllowEncodedSlashes NoDecode
        
 </VirtualHost>
 
 
+#------------------------
+# Catch-all
+#------------------------
 <VirtualHost *:443>
     ServerAdmin admin@rosetta.platform
     SSLEngine on
-    SSLCertificateFile /root/certificates/rosetta_platform/rosetta_platform.crt
-    SSLCertificateKeyFile /root/certificates/rosetta_platform/rosetta_platform.key
-    SSLCACertificateFile /root/certificates/rosetta_platform/rosetta_platform.ca-bundle
+    SSLCertificateFile /etc/letsencrypt/live/__ROSETTA_HOST__/cert.pem
+    SSLCertificateKeyFile /etc/letsencrypt/live/__ROSETTA_HOST__/privkey.pem
+    SSLCACertificateFile /etc/letsencrypt/live/__ROSETTA_HOST__/fullchain.pem
     DocumentRoot /var/www/html
 </VirtualHost>
 
 
+#------------------------
+#  Rosetta Platform
+#------------------------
 <VirtualHost *:443>
     ServerAdmin admin@rosetta.platform
-    ServerName ${ROSETTA_HOST}
+    ServerName __ROSETTA_HOST__
     ProxyPass / http://webapp:8080/
     ProxyPassReverse / http://webapp:8080/
     AllowEncodedSlashes NoDecode
 
     SSLEngine on
-    SSLCertificateFile /root/certificates/rosetta_platform/rosetta_platform.crt
-    SSLCertificateKeyFile /root/certificates/rosetta_platform/rosetta_platform.key
-    SSLCACertificateFile /root/certificates/rosetta_platform/rosetta_platform.ca-bundle
+    SSLCertificateFile /etc/letsencrypt/live/__ROSETTA_HOST__/cert.pem
+    SSLCertificateKeyFile /etc/letsencrypt/live/__ROSETTA_HOST__/privkey.pem
+    SSLCACertificateFile /etc/letsencrypt/live/__ROSETTA_HOST__/fullchain.pem
 
     # Browser-specific fixes
     BrowserMatch "MSIE [2-6]" \
@@ -51,18 +59,40 @@
 
 </VirtualHost>
 
+
+#------------------------
+#  Rosetta tasks
+#------------------------
+# This is actually a placeholder required in order to have Let's Encrypt to get the
+# certificates, as Apache conf fot hte tasks is injected by the webapp service.
+# If __ROSETTA_TASKS_PROXY_HOST__ is set qual to __ROSETTA_HOST__ as no dual
+# configuration is in place, then it simply gets overwritten by the entry above.
+
+<VirtualHost *:443>
+    ServerAdmin admin@rosetta.platform
+    ServerName __ROSETTA_TASKS_PROXY_HOST__
+    SSLEngine on
+    SSLCertificateFile /etc/letsencrypt/live/__ROSETTA_TASKS_PROXY_HOST__/cert.pem
+    SSLCertificateKeyFile /etc/letsencrypt/live/__ROSETTA_TASKS_PROXY_HOST__/privkey.pem
+    SSLCACertificateFile /etc/letsencrypt/live/__ROSETTA_TASKS_PROXY_HOST__/fullchain.pem
+    DocumentRoot /var/www/html
+</VirtualHost>
+
+
+#-------------------------
+#  The Docker registry
+#-------------------------
 Listen 5000
 <VirtualHost *:5000>
     ServerAdmin admin@rosetta.platform
-    #ServerName ${ROSETTA_HOST}
     ProxyPass / http://dregistry:5000/
     ProxyPassReverse / http://dregistry:5000/
 
     SSLEngine on
 
-    SSLCertificateFile /root/certificates/rosetta_platform/rosetta_platform.crt
-    SSLCertificateKeyFile /root/certificates/rosetta_platform/rosetta_platform.key
-    SSLCACertificateFile /root/certificates/rosetta_platform/rosetta_platform.ca-bundle
+    SSLCertificateFile /etc/letsencrypt/live/__ROSETTA_HOST__/cert.pem
+    SSLCertificateKeyFile /etc/letsencrypt/live/__ROSETTA_HOST__/privkey.pem
+    SSLCACertificateFile /etc/letsencrypt/live/__ROSETTA_HOST__/fullchain.pem
 
     # Browser-specific fixes
     BrowserMatch "MSIE [2-6]" \

services/proxy/run_certbot.sh

+#!/bin/bash
+
+# Source env
+source /env.sh
+
+# Exec certbot renew every hour
+while true
+do
+    date
+    sudo certbot renew
+    sleep 86400
+done

services/proxy/supervisord_apache.conf

@@ -5,7 +5,7 @@
 [program:apache]
  
 ; General
-command       = /etc/supervisor/conf.d/run_Apache.sh
+command       = /etc/supervisor/conf.d/run_apache.sh
 user          = root
 numprocs      = 1
 autostart     = true
@@ -14,10 +14,8 @@ startsecs     = 10
 stopwaitsecs  = 30
 process_name  = apache
  
-; Standard out / error
-stdout_logfile          = /var/log/supervisor/%(program_name)s.log
-stdout_logfile_maxbytes = 5MB
+; Log files
+stdout_logfile          = /var/log/proxy/apache.log
+stdout_logfile_maxbytes = 10MB
 stdout_logfile_backups  = 10
-stderr_logfile          = /var/log/supervisor/%(program_name)s.log
-stderr_logfile_maxbytes = 5MB
-stderr_logfile_backups  = 10
+redirect_stderr         = true

services/proxy/supervisord_certbot.conf

+;=======================================
+; Certbot service
+;=======================================
+ 
+[program:certbot]
+ 
+; General
+command       = /etc/supervisor/conf.d/run_certbot.sh
+user          = root
+numprocs      = 1
+autostart     = true
+autorestart   = true
+startsecs     = 10
+stopwaitsecs  = 30
+process_name  = certbot
+ 
+; Log files
+stdout_logfile          = /var/log/proxy/certbot.log
+stdout_logfile_maxbytes = 10MB
+stdout_logfile_backups  = 10
+redirect_stderr         = true

services/webapp/Dockerfile

@@ -19,10 +19,6 @@ RUN curl -O https://bootstrap.pypa.io/pip/3.6/get-pip.py
 # Install Python3 and Pip3 (python3-distutils required for pip3)
 RUN apt-get install python3 python3-distutils -y 
 
-# Install Python and pip in this order (first Python 3 and then Python 2), or 
-# you will end ap with python defaulting to python2 and pip defaulting to pip3
-# Otherwise, do somethign like "ln -s /usr/local/bin/pip3 /usr/local/bin/pip"
-
 # Install Python3 and Pip3 (ython3-distutils required for pip3)
 RUN apt-get install python3 python3-distutils -y 
 RUN python3 get-pip.py 'pip==21.0.1'

services/webapp/code/rosetta/core_app/models.py

@@ -5,7 +5,7 @@ from django.conf import settings
 from django.db import models
 from django.contrib.auth.models import User, Group
 from django.utils import timezone
-from .utils import os_shell, color_map, hash_string_to_int, get_task_tunnel_host
+from .utils import os_shell, color_map, hash_string_to_int, get_rosetta_tasks_tunnel_host
 from .exceptions import ConsistencyException
 
 if 'sqlite' in settings.DATABASES['default']['ENGINE']:
@@ -356,7 +356,7 @@ class Task(models.Model):
     
     @property
     def tcp_tunnel_host(self):
-        return get_task_tunnel_host()
+        return get_rosetta_tasks_tunnel_host()
 
 
 

services/webapp/code/rosetta/core_app/utils.py

@@ -515,12 +515,16 @@ def get_platform_registry():
     platform_registry_conn_string = '{}:{}'.format(platform_registry_host, platform_registry_port)
     return platform_registry_conn_string
     
-def get_task_tunnel_host():
-    tunnel_host = os.environ.get('TASK_TUNNEL_HOST', 'localhost')
+def get_rosetta_tasks_tunnel_host():
+    # Importing here instead of on top avoids circular dependencies problems when loading booleanize in settings
+    from django.conf import settings
+    tunnel_host = os.environ.get('ROSETTA_TASKS_TUNNEL_HOST', settings.ROSETTA_HOST)
     return tunnel_host
 
-def get_task_proxy_host():
-    proxy_host = os.environ.get('TASK_PROXY_HOST', 'localhost')
+def get_rosetta_tasks_proxy_host():
+    # Importing here instead of on top avoids circular dependencies problems when loading booleanize in settings
+    from django.conf import settings
+    proxy_host = os.environ.get('ROSETTA_TASKS_PROXY_HOST', settings.ROSETTA_HOST)
     return proxy_host
 
 def hash_string_to_int(string):
@@ -622,7 +626,7 @@ def setup_tunnel_and_proxy(task):
             # Some info about the various SSL switches: https://serverfault.com/questions/577616/using-https-between-apache-loadbalancer-and-backends
             logger.debug('Writing task proxy conf to {}'.format(apache_conf_file))
             websocket_protocol = 'wss' if task.container.interface_protocol == 'https' else 'ws'
-            task_proxy_host = get_task_proxy_host()
+            rosetta_tasks_proxy_host = get_rosetta_tasks_proxy_host()
             apache_conf_content = '''
 #---------------------------
 #  Task interface proxy 
@@ -641,7 +645,7 @@ Listen '''+str(task.tcp_tunnel_port)+'''
 
 <VirtualHost *:'''+str(task.tcp_tunnel_port)+'''>
     
-    ServerName  '''+task_proxy_host+'''
+    ServerName  '''+rosetta_tasks_proxy_host+'''
     ServerAdmin admin@rosetta.platform
     
     SSLEngine on

services/webapp/code/rosetta/core_app/views.py

@@ -13,8 +13,8 @@ from django.contrib.auth.models import User
 from django.shortcuts import redirect
 from django.db.models import Q
 from .models import Profile, LoginToken, Task, TaskStatuses, Container, Computing, KeyPair, Page
-from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_task_tunnel_host
-from .utils import get_task_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation
+from .utils import send_email, format_exception, timezonize, os_shell, booleanize, get_rosetta_tasks_tunnel_host
+from .utils import get_rosetta_tasks_proxy_host, random_username, setup_tunnel_and_proxy, finalize_user_creation
 from .utils import sanitize_container_env_vars, get_or_create_container_from_repository
 from .decorators import public_view, private_view
 from .exceptions import ErrorMessage
@@ -1183,19 +1183,19 @@ def direct_connection_handler(request, uuid):
     setup_tunnel_and_proxy(task)
     
     # Get task and tunnel proxy host
-    task_proxy_host = get_task_proxy_host()
-    task_tunnel_host = get_task_tunnel_host()
+    rosetta_tasks_proxy_host = get_rosetta_tasks_proxy_host()
+    rosetta_tasks_tunnel_host = get_rosetta_tasks_tunnel_host()
 
     # Redirect to the task through the tunnel    
     if task.requires_proxy:
         if task.requires_proxy_auth and task.auth_token:
             user = request.user.email
             password = task.auth_token
-            redirect_string = 'https://{}:{}@{}:{}'.format(user, password, task_proxy_host, task.tcp_tunnel_port)        
+            redirect_string = 'https://{}:{}@{}:{}'.format(user, password, rosetta_tasks_proxy_host, task.tcp_tunnel_port)        
         else:
-            redirect_string = 'https://{}:{}'.format(task_proxy_host, task.tcp_tunnel_port)       
+            redirect_string = 'https://{}:{}'.format(rosetta_tasks_proxy_host, task.tcp_tunnel_port)       
     else:
-        redirect_string = '{}://{}:{}'.format(task.container.interface_protocol, task_tunnel_host, task.tcp_tunnel_port)
+        redirect_string = '{}://{}:{}'.format(task.container.interface_protocol, rosetta_tasks_tunnel_host, task.tcp_tunnel_port)
     
     logger.debug('Task direct connect redirect: "{}"'.format(redirect_string))
     return redirect(redirect_string)
@@ -1216,14 +1216,14 @@ def sharable_link_handler(request, short_uuid):
     setup_tunnel_and_proxy(task)
     
     # Get task and tunnel proxy host
-    task_proxy_host = get_task_proxy_host()
-    task_tunnel_host = get_task_tunnel_host()
+    rosetta_tasks_proxy_host = get_rosetta_tasks_proxy_host()
+    rosetta_tasks_tunnel_host = get_rosetta_tasks_tunnel_host()
 
     # Redirect to the task through the tunnel    
     if task.requires_proxy:
-        redirect_string = 'https://{}:{}'.format(task_proxy_host, task.tcp_tunnel_port)       
+        redirect_string = 'https://{}:{}'.format(rosetta_tasks_proxy_host, task.tcp_tunnel_port)       
     else:
-        redirect_string = '{}://{}:{}'.format(task.container.interface_protocol, task_tunnel_host, task.tcp_tunnel_port)
+        redirect_string = '{}://{}:{}'.format(task.container.interface_protocol, rosetta_tasks_tunnel_host, task.tcp_tunnel_port)
     
     logger.debug('Task sharable link connect redirect: "{}"'.format(redirect_string))
     return redirect(redirect_string)