Skip to content
Commits on Source (8)
ddf2/DDFacet/
ddf2/killMS/
ddf2/keys
.DS_Store
.project
.pydevproject
FROM lofar/lofar-pipeline:LOFAR-Release-4_0_16
# Set non-interactive
ENV DEBIAN_FRONTEND noninteractive
# Always update when extending base images
RUN apt update
#------------------------
# Install deps
#------------------------
# Git, Curl, sudo and Nano
RUN apt-get install git curl sudo nano -y
#------------------------
# Lofar user
#------------------------
# Add group. We chose GID 65527 to try avoiding conflicts.
RUN groupadd -g 65527 lofar
# Add user. We chose UID 65527 to try avoiding conflicts.
RUN useradd lofar -d /home/lofar -u 65527 -g 65527 -m -s /bin/bash
# Add metuaser user to sudoers
RUN adduser lofar sudo
# No pass sudo (for everyone, actually)
COPY sudoers /etc/sudoers
# Setup home and input/output data directories
COPY data/input_data /input_data
COPY data/output_data /output_data
RUN chown lofar:lofar /home/lofar && chown -R lofar:lofar /input_data && chown -R lofar:lofar /output_data
# This is required mainly for Singularity
RUN mv /home/lofar /home/vanilla_lofar
RUN ln -s /tmp/lofarhome /home/lofar
RUN rm -rf /tmp/lofarhome
#----------------------
# Entrypoint
#----------------------
# Copy entrypoint
COPY entrypoint.sh /
# Give right permissions
RUN chmod 755 /entrypoint.sh
# Set entrypoint
ENTRYPOINT ["/entrypoint.sh"]
# Set user lofar
USER lofar
# Set container name
ENV CONTAINER_NAME='base4.0.16'
#!/bin/bash
docker build . -t lofarit/base4.0.16
Put here "calib" and "target" MS data (or link agains a folder containing them).
Directory for output data.
#!/bin/bash
# Exit on any error. More complex thing could be done in future
# (see https://stackoverflow.com/questions/4381618/exit-a-script-on-error)
set -e
echo ""
echo "[INFO] Executing entrypoint..."
#echo "[INFO] Sourcing env in /opt/lofarsoft/lofarinit.sh..."
#source /opt/lofarsoft/lofarinit.sh
echo "[INFO] Creating /tmp/lofarhome to be used as lofar home"
mkdir /tmp/lofarhome
echo "[INFO] Initialising /tmp/lofarhome with configuration files"
cp -aT /home/vanilla_lofar /tmp/lofarhome
echo "[INFO] Moving to /home/lofar and setting as home"
cd /home/lofar
export HOME=/home/lofar
echo "[INFO] Setting new prompt @$CONTAINER_NAME container"
echo 'export PS1="${debian_chroot:+($debian_chroot)}\u@$CONTAINER_NAME@\h:\w\$ "' >> /tmp/lofarhome/.bashrc
# Set entrypoint command
if [ "x$@" == "x" ]; then
COMMAND="/bin/bash"
else
COMMAND="$@"
fi
# Start!
echo -n "[INFO] Will execute entrypoint command: "
echo $COMMAND
echo ""
echo "=============================================================="
echo ""
echo " Welcome to the LOFAR-IT $CONTAINER_NAME container!"
echo ""
echo "=============================================================="
echo ""
echo "You are now in /home/lofar with write access as user \"$(whoami)\"."
echo ""
echo "Remember that contents inside this container, unless stored"
echo "on a persistent volume mounted from you host machine, will"
echo "be wiped out when exiting the container."
echo ""
exec $COMMAND
#!/bin/bash
docker run --rm -v $PWD/data:/data -it lofarit/base4.0.16 /bin/bash
#
# This file MUST be edited with the 'visudo' command as root.
#
# Please consider adding local content in /etc/sudoers.d/ instead of
# directly modifying this file.
#
# See the man page for details on how to write a sudoers file.
#
Defaults env_reset
Defaults mail_badpass
Defaults secure_path="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
# Host alias specification
# User alias specification
# Cmnd alias specification
# User privilege specification
root ALL=(ALL:ALL) ALL
# Members of the admin group may gain root privileges
%admin ALL=(ALL) ALL
# Allow members of group sudo to execute any command
%sudo ALL=(ALL:ALL) NOPASSWD:ALL
# See sudoers(5) for more information on "#include" directives:
#includedir /etc/sudoers.d
FROM lofarit/base3.10
FROM lofarit/base4.0.16
USER root
RUN apt-get install llvm-7 -y
ENV LLVM_CONFIG=llvm-config-7
ENV PYTHONPATH /opt/lofarsoft/lib/python2.7/site-packages
#------------------------
# Get and install ddf2
#------------------------
# DDF pipeline
RUN cd /opt && git clone https://github.com/mhardcastle/ddf-pipeline.git
RUN cd /opt/ddf-pipeline && git checkout fdaa5aa
# SkyModel
RUN cd /opt && git clone https://github.com/cyriltasse/SkyModel.git
RUN cd /opt/SkyModel && git checkout 026997f
# killMS
COPY killMS /opt/killMS
#RUN cd /opt/killMS && python setup.py build
RUN cd /opt/killMS/Predict && make
RUN cd /opt/killMS/Array/Dot && make
RUN cd /opt/killMS/Gridder && make
# DynSpecMS
RUN cd /opt && git clone https://github.com/cyriltasse/DynSpecMS.git
RUN cd /opt/DynSpecMS && git checkout 461183f
# DDFacet
COPY DDFacet /opt/DDFacet
RUN cp /opt/ddf-pipeline/misc/setup.cfg /opt/DDFacet/ # Fix compile options
RUN python -m pip install -U pip setuptools wheel
RUN pip install -U pip setuptools wheel
RUN python -m pip install numpy==1.16.2 bdsf==1.8.15 emcee
RUN python -m pip install astropy_healpix pybind11 future pyregion sshtunnel pymysql psutil
RUN python -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"
RUN cd /opt/DDFacet && rm -rf /opt/DDFacet/Dcbuild && python setup.py build
RUN python -m pip install bdsf==1.8.15 numpy==1.16.2
RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh
# Package dependencies
COPY apt.sources.list /etc/apt/sources.list
# Support large mlocks
RUN echo "* - memlock unlimited" > /etc/security/limits.conf
ENV DEBIAN_FRONTEND noninteractive
ENV DEBIAN_PRIORITY critical
ENV GNUCOMPILER 7
ENV DEB_SETUP_DEPENDENCIES \
dpkg-dev \
g++-$GNUCOMPILER \
gcc-$GNUCOMPILER \
libc-dev \
cmake \
gfortran-$GNUCOMPILER \
git \
wget \
subversion
ENV DEB_DEPENCENDIES \
python3-pip \
libfftw3-dev \
python3-numpy \
libfreetype6 \
libfreetype6-dev \
libpng-dev \
pkg-config \
python3-dev \
libboost-all-dev \
libcfitsio-dev \
libhdf5-dev \
wcslib-dev \
libatlas-base-dev \
liblapack-dev \
python3-tk \
libreadline6-dev \
subversion \
liblog4cplus-dev \
libhdf5-dev \
libncurses5-dev \
libsofa1-dev \
flex \
bison \
libbison-dev \
# Reference image generation dependencies
make
RUN apt-get update
RUN apt-get install -y $DEB_SETUP_DEPENDENCIES
RUN apt-get install -y $DEB_DEPENCENDIES
ENV PATH /usr/local/bin:$PATH
ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH
ENV PYTHONPATH /usr/local/lib/python2.7/site-packages:$PYTHONPATH
# Latest GCC segfaults when compiling casacore
RUN rm /usr/bin/gcc /usr/bin/g++ /usr/bin/cpp /usr/bin/cc
RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/gcc
RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/g++
RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/cc
RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp
RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran
#####################################################################
# Add deployment keys
#####################################################################
RUN mkdir -p /root/.ssh/
COPY keys/lofarit_ddf2_rsa /root/.ssh/lofarit_ddf2_rsa
COPY keys/lofarit_killms_rsa /root/.ssh/lofarit_killms_rsa
RUN chmod 600 /root/.ssh/lofarit_ddf2_rsa && chmod 600 /root/.ssh/lofarit_killms_rsa
#####################################################################
# Get DDF from private repo
#####################################################################
# Add github key(s)
RUN ssh-keyscan github.com >> ~/.ssh/known_hosts
# Clone
RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/ && git clone git@github.com:cyriltasse/DDFacet'
# Checkout
RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/DDFacet && git pull && git checkout e98968d'
#####################################################################
# Build DDF from source
#####################################################################
RUN mkdir /opt/DDFacet
RUN sed 's/numpy (<=1.16)/numpy (<=1.16.2)/g' /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA > /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA
# Copy DDFacet and SkyModel into the image
RUN cp -a /root/DDFacet/DDFacet /opt/DDFacet/DDFacet && \
cp -a /root/DDFacet/SkyModel /opt/DDFacet/SkyModel && \
cp -a /root/DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in && \
cp -a /root/DDFacet/setup.py /opt/DDFacet/setup.py && \
cp -a /root/DDFacet/setup.cfg /opt/DDFacet/setup.cfg && \
cp -a /root/DDFacet/README.rst /opt/DDFacet/README.rst && \
cp -a /root/DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml && \
cp -a /root/DDFacet/.git /opt/DDFacet/.git && \
cp -a /root/DDFacet/.gitignore /opt/DDFacet/.gitignore && \
cp -a /root/DDFacet/.gitmodules /opt/DDFacet/.gitmodules
# Finally install DDFacet
RUN rm -rf /opt/DDFacet/DDFacet/cbuild
RUN pip3 install -U pip setuptools wheel
RUN python3 -m pip install pybind11
RUN python3 -m pip install tensorflow==1.8.0
RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"
RUN cd /opt/DDFacet/ && python3 setup.py build && cd /
# Set MeqTrees Cattery path to installation directory
ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/
ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH
RUN python3 -m pip install numpy==1.17.0
RUN python3 -c "import Siamese"
RUN python3 -c "import bdsf"
# Perform some basic tests
RUN DDF.py --help
RUN MakeMask.py --help
RUN MakeCatalog.py --help
RUN MakeModel.py --help
RUN MaskDicoModel.py --help
RUN ClusterCat.py --help
#####################################################################
# Get and build killMs
#####################################################################
# Clone
RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/ && git clone git@github.com:cyriltasse/killMS'
# Checkout
RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/killMS && git pull && git checkout 0607798'
RUN pip3 install /opt/killMS
RUN cd /opt/killMS; python3 setup.py build
#####################################################################
# Remove deploymnet keys
#####################################################################
RUN rm -f /root/.ssh/lofarit_ddf2_rsa && rm -f /root/.ssh/lofarit_killms_rsa
#####################################################################
# DDF-PIPELINE
#####################################################################
RUN pip3 install pyregion sshtunnel pymysql
RUN cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline
RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh
RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git
#USER lofar
ENV CONTAINER_NAME='ddf2_base3.10'
#####################################################################
# Switch back to lofar user and set env
#####################################################################
USER lofar
ENV DDF_PIPELINE_CATALOGS='/opt/container-data/bootstrap-cats/'
ENV CONTAINER_NAME='ddfacet_ssd-py3_kms-lofar-release-4_0_16'
###### Ubuntu Main Repos
deb http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse
deb-src http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse
###### Ubuntu Update Repos
deb http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse
deb http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse
deb http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse
deb http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse
deb-src http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse
deb-src http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse
deb-src http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse
deb-src http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse
#!/bin/bash
docker build . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16
#!/bin/bash
set -e
rm -rf DDFacet
git clone https://github.com/cyriltasse/DDFacet.git
git checkout 1059028
docker build . -t lofarit/ddf2_base3.10