.gitignore

-
-ddf2/DDFacet/
-ddf2/killMS/
+ddf2/keys
+.DS_Store
+.project
+.pydevproject

base_v4/Dockerfile

+FROM lofar/lofar-pipeline:LOFAR-Release-4_0_16
+
+# Set non-interactive
+ENV DEBIAN_FRONTEND noninteractive
+
+# Always update when extending base images
+RUN apt update
+
+
+#------------------------
+# Install deps
+#------------------------
+
+# Git, Curl, sudo and  Nano
+RUN apt-get install git curl sudo nano -y
+
+
+#------------------------
+# Lofar user
+#------------------------
+
+# Add group. We chose GID 65527 to try avoiding conflicts.
+RUN groupadd -g 65527 lofar
+
+# Add user. We chose UID 65527 to try avoiding conflicts.
+RUN useradd lofar -d /home/lofar -u 65527 -g 65527 -m -s /bin/bash
+
+# Add metuaser user to sudoers
+RUN adduser lofar sudo
+
+# No pass sudo (for everyone, actually)
+COPY sudoers /etc/sudoers
+
+# Setup home and input/output data directories
+COPY data/input_data /input_data
+COPY data/output_data /output_data
+RUN chown lofar:lofar /home/lofar && chown -R lofar:lofar /input_data && chown -R lofar:lofar /output_data
+
+# This is required mainly for Singularity
+RUN mv /home/lofar /home/vanilla_lofar
+RUN ln -s /tmp/lofarhome /home/lofar
+RUN rm -rf /tmp/lofarhome
+
+
+#----------------------
+# Entrypoint
+#----------------------
+
+# Copy entrypoint
+COPY entrypoint.sh /
+
+# Give right permissions
+RUN chmod 755 /entrypoint.sh
+
+# Set entrypoint
+ENTRYPOINT ["/entrypoint.sh"]
+
+# Set user lofar
+USER lofar
+
+# Set container name
+ENV CONTAINER_NAME='base4.0.16'

base_v4/build.sh

+#!/bin/bash
+
+docker build  . -t lofarit/base4.0.16

base_v4/data/input_data/README.md

+Put here "calib" and "target" MS data (or link agains a folder containing them).

base_v4/data/output_data/README.md

+Directory for output data.

base_v4/entrypoint.sh

+#!/bin/bash
+
+# Exit on any error. More complex thing could be done in future
+# (see https://stackoverflow.com/questions/4381618/exit-a-script-on-error)
+set -e
+
+echo ""
+echo "[INFO] Executing entrypoint..."
+
+#echo "[INFO] Sourcing env in /opt/lofarsoft/lofarinit.sh..."
+#source /opt/lofarsoft/lofarinit.sh
+
+echo "[INFO] Creating /tmp/lofarhome to be used as lofar home"
+mkdir /tmp/lofarhome
+
+echo "[INFO] Initialising /tmp/lofarhome with configuration files"
+cp -aT /home/vanilla_lofar /tmp/lofarhome
+
+echo "[INFO] Moving to /home/lofar and setting as home"
+cd /home/lofar
+export HOME=/home/lofar
+
+echo "[INFO] Setting new prompt @$CONTAINER_NAME container"
+echo 'export PS1="${debian_chroot:+($debian_chroot)}\u@$CONTAINER_NAME@\h:\w\$ "' >> /tmp/lofarhome/.bashrc
+
+# Set entrypoint command
+if [ "x$@" == "x" ]; then
+    COMMAND="/bin/bash"
+else
+    COMMAND="$@"
+fi
+
+
+
+# Start!
+echo -n "[INFO] Will execute entrypoint command: "
+echo $COMMAND
+echo ""
+echo "=============================================================="
+echo ""
+echo "      Welcome to the LOFAR-IT $CONTAINER_NAME container!"
+echo ""
+echo "=============================================================="
+echo ""
+echo "You are now in /home/lofar with write access as user \"$(whoami)\"."
+echo ""
+echo "Remember that contents inside this container, unless stored"
+echo "on a persistent volume mounted from you host machine, will"
+echo "be wiped out when exiting the container."
+echo ""
+
+exec $COMMAND
+

base_v4/run.sh

+#!/bin/bash
+docker run --rm -v $PWD/data:/data -it lofarit/base4.0.16 /bin/bash

base_v4/sudoers

+#
+# This file MUST be edited with the 'visudo' command as root.
+#
+# Please consider adding local content in /etc/sudoers.d/ instead of
+# directly modifying this file.
+#
+# See the man page for details on how to write a sudoers file.
+#
+Defaults        env_reset
+Defaults        mail_badpass
+Defaults        secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
+
+# Host alias specification
+
+# User alias specification
+
+# Cmnd alias specification
+
+# User privilege specification
+root    ALL=(ALL:ALL) ALL
+
+# Members of the admin group may gain root privileges
+%admin ALL=(ALL) ALL
+
+# Allow members of group sudo to execute any command
+%sudo   ALL=(ALL:ALL) NOPASSWD:ALL
+
+# See sudoers(5) for more information on "#include" directives:
+
+#includedir /etc/sudoers.d

ddf2/Dockerfile

-FROM lofar/lofar-pipeline:LOFAR-Release-4_0_16
+FROM lofarit/base4.0.16
+
+USER root
 
 # Package dependencies
 COPY apt.sources.list /etc/apt/sources.list
@@ -65,21 +67,48 @@ RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp
 RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran
 
 
+#####################################################################
+# Add deployment keys
+#####################################################################
+
+RUN mkdir -p /root/.ssh/ 
+COPY keys/lofarit_ddf2_rsa /root/.ssh/lofarit_ddf2_rsa
+COPY keys/lofarit_killms_rsa /root/.ssh/lofarit_killms_rsa
+RUN chmod 600 /root/.ssh/lofarit_ddf2_rsa && chmod 600 /root/.ssh/lofarit_killms_rsa
+
+
+#####################################################################
+# Get DDF from private repo
+#####################################################################
+
+# Add github key(s)
+RUN ssh-keyscan github.com >> ~/.ssh/known_hosts
+
+# Clone
+RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/ && git clone git@github.com:cyriltasse/DDFacet'
+
+# Checkout
+RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/DDFacet && git pull && git checkout e98968d'
+
+
 
 #####################################################################
-## BUILD DDF FROM SOURCE
+# Build DDF from source
 #####################################################################
+
+RUN mkdir /opt/DDFacet
+
 # Copy DDFacet and SkyModel into the image
-ADD DDFacet/DDFacet /opt/DDFacet/DDFacet
-ADD DDFacet/SkyModel /opt/DDFacet/SkyModel
-ADD DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in
-ADD DDFacet/setup.py /opt/DDFacet/setup.py
-ADD DDFacet/setup.cfg /opt/DDFacet/setup.cfg
-ADD DDFacet/README.rst /opt/DDFacet/README.rst
-ADD DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml
-ADD DDFacet/.git /opt/DDFacet/.git
-ADD DDFacet/.gitignore /opt/DDFacet/.gitignore
-ADD DDFacet/.gitmodules /opt/DDFacet/.gitmodules
+RUN cp -a /root/DDFacet/DDFacet /opt/DDFacet/DDFacet && \
+    cp -a /root/DDFacet/SkyModel /opt/DDFacet/SkyModel && \
+    cp -a /root/DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in && \
+    cp -a /root/DDFacet/setup.py /opt/DDFacet/setup.py && \
+    cp -a /root/DDFacet/setup.cfg /opt/DDFacet/setup.cfg && \
+    cp -a /root/DDFacet/README.rst /opt/DDFacet/README.rst && \
+    cp -a /root/DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml && \
+    cp -a /root/DDFacet/.git /opt/DDFacet/.git && \
+    cp -a /root/DDFacet/.gitignore /opt/DDFacet/.gitignore && \
+    cp -a /root/DDFacet/.gitmodules /opt/DDFacet/.gitmodules
 
 
 # Finally install DDFacet
@@ -89,17 +118,17 @@ RUN python3 -m pip install pybind11
 RUN python3 -m pip install tensorflow==1.8.0
 RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"
 RUN cd /opt/DDFacet/ && python3 setup.py build && cd /
+
 # Set MeqTrees Cattery path to installation directory
 ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/
 ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH
 
 RUN python3 -m pip install numpy==1.17.0
 
-
 RUN python3 -c "import Siamese"
 RUN python3 -c "import bdsf"
 
-# perform some basic tests
+# Perform some basic tests
 RUN DDF.py --help
 RUN MakeMask.py --help
 RUN MakeCatalog.py --help
@@ -107,15 +136,30 @@ RUN MakeModel.py --help
 RUN MaskDicoModel.py --help
 RUN ClusterCat.py --help
 
+
 #####################################################################
-## killMs
+# Get and build killMs
 #####################################################################
-ADD killMS /opt/killMS
+
+# Clone
+RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/ && git clone git@github.com:cyriltasse/killMS'
+
+# Checkout
+RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/killMS && git pull && git checkout 0607798'
+
 RUN pip3 install /opt/killMS
 RUN cd  /opt/killMS; python3 setup.py build
 
+
+#####################################################################
+# Remove deploymnet keys
+#####################################################################
+
+RUN rm -f /root/.ssh/lofarit_ddf2_rsa && rm -f /root/.ssh/lofarit_killms_rsa
+
+
 #####################################################################
-## DDF-PIPELINE
+# DDF-PIPELINE
 #####################################################################
 RUN pip3 install pyregion sshtunnel pymysql
 RUN  cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline
@@ -123,5 +167,12 @@ RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/ini
 
 RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git
 
+
+#####################################################################
+# Switch back to lofar user and set env
+#####################################################################
+
+USER lofar
+
 ENV DDF_PIPELINE_CATALOGS='/opt/container-data/bootstrap-cats/'
 ENV CONTAINER_NAME='ddfacet_ssd-py3_kms-lofar-release-4_0_16'

ddf2/build_docker_container.sh → ddf2/build.sh

 #!/bin/bash
-set -e 
-rm -rf DDFacet
-rm -rf killMS
-
-git clone https://github.com/cyriltasse/killMS.git
-git checkout 0607798
-
-
-git clone https://github.com/cyriltasse/DDFacet.git
-git checkout e98968d
-
 
 docker build  . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16

ddf2/run.sh

+#!/bin/bash
+docker run --rm -v $PWD/data:/data -it lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16 /bin/bash

ddf2/run_docker_container.sh

-#!/bin/bash
-docker run --rm -v $PWD/data:/data -it lofarit/ddf2_base3.10 /bin/bash