Commit d2987e9f authored by Fabio Roberto Vitello's avatar Fabio Roberto Vitello
Browse files

Updated version of DDFacet pipeline 

This container employ:
- lofar-release-4_0_16
- ddfacet branch ssd-py3_kms
parent 4183dc87

ddf2/Dockerfile

+122 −47
Original line number Diff line number Diff line 
FROM lofarit/base3.10



USER root



RUN apt-get install llvm-7 -y

ENV LLVM_CONFIG=llvm-config-7 

ENV PYTHONPATH /opt/lofarsoft/lib/python2.7/site-packages

#------------------------

# Get and install ddf2

#------------------------

# DDF pipeline

RUN cd /opt && git clone https://github.com/mhardcastle/ddf-pipeline.git

RUN cd /opt/ddf-pipeline && git checkout fdaa5aa



# SkyModel

RUN cd /opt && git clone https://github.com/cyriltasse/SkyModel.git

RUN cd /opt/SkyModel && git checkout 026997f



# killMS

COPY killMS /opt/killMS

#RUN cd /opt/killMS && python setup.py build

RUN cd /opt/killMS/Predict && make

RUN cd /opt/killMS/Array/Dot && make

RUN cd /opt/killMS/Gridder && make



# DynSpecMS

RUN cd /opt && git clone https://github.com/cyriltasse/DynSpecMS.git

RUN cd /opt/DynSpecMS && git checkout 461183f



# DDFacet

COPY DDFacet /opt/DDFacet

RUN cp /opt/ddf-pipeline/misc/setup.cfg /opt/DDFacet/ # Fix compile options

RUN python -m pip install -U pip setuptools wheel

RUN pip install -U pip setuptools wheel

RUN python -m pip install numpy==1.16.2 bdsf==1.8.15 emcee



RUN python -m pip install astropy_healpix pybind11 future pyregion sshtunnel pymysql psutil

RUN python -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"



RUN cd /opt/DDFacet && rm -rf /opt/DDFacet/Dcbuild && python setup.py build



RUN python -m pip install bdsf==1.8.15 numpy==1.16.2

RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh

FROM lofar/lofar-pipeline:LOFAR-Release-4_0_16



RUN  sed 's/numpy (<=1.16)/numpy (<=1.16.2)/g' /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA >  /usr/local/lib/python2.7/dist-packages/meqtrees_cattery-1.7.0.dist-info/METADATA

#Package dependencies

COPY apt.sources.list /etc/apt/sources.list



RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git

# Support large mlocks

RUN echo "*        -   memlock     unlimited" > /etc/security/limits.conf



ENV DEBIAN_FRONTEND noninteractive

ENV DEBIAN_PRIORITY critical

ENV GNUCOMPILER 7

ENV DEB_SETUP_DEPENDENCIES \

    dpkg-dev \

    g++-$GNUCOMPILER \

    gcc-$GNUCOMPILER \

    libc-dev \

    cmake \

    gfortran-$GNUCOMPILER \

    git \

    wget \

    subversion



ENV DEB_DEPENCENDIES \

    python3-pip \

    libfftw3-dev \

    python3-numpy \

    libfreetype6 \

    libfreetype6-dev \

    libpng-dev \

    pkg-config \

    python3-dev \

    libboost-all-dev \

    libcfitsio-dev \

    libhdf5-dev \

    wcslib-dev \

    libatlas-base-dev \

    liblapack-dev \

    python3-tk \

    libreadline6-dev \

    subversion \

    liblog4cplus-dev \

    libhdf5-dev \

    libncurses5-dev \

    libsofa1-dev \

    flex \

    bison \

    libbison-dev \

    # Reference image generation dependencies

    make



RUN apt-get update

RUN apt-get install -y $DEB_SETUP_DEPENDENCIES

RUN apt-get install -y $DEB_DEPENCENDIES



ENV PATH /usr/local/bin:$PATH

ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH

ENV PYTHONPATH /usr/local/lib/python2.7/site-packages:$PYTHONPATH



# Latest GCC segfaults when compiling casacore

RUN rm /usr/bin/gcc /usr/bin/g++ /usr/bin/cpp /usr/bin/cc

RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/gcc

RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/g++

RUN ln -s /usr/bin/gcc-$GNUCOMPILER /usr/bin/cc

RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp

RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran







#####################################################################

## BUILD DDF FROM SOURCE

#####################################################################

#Copy DDFacet and SkyModel into the image

ADD DDFacet/DDFacet /opt/DDFacet/DDFacet

ADD DDFacet/SkyModel /opt/DDFacet/SkyModel

ADD DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in

ADD DDFacet/setup.py /opt/DDFacet/setup.py

ADD DDFacet/setup.cfg /opt/DDFacet/setup.cfg

ADD DDFacet/README.rst /opt/DDFacet/README.rst

ADD DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml

ADD DDFacet/.git /opt/DDFacet/.git

ADD DDFacet/.gitignore /opt/DDFacet/.gitignore

ADD DDFacet/.gitmodules /opt/DDFacet/.gitmodules



#USER lofar



ENV CONTAINER_NAME='ddf2_base3.10'

# Finally install DDFacet

RUN rm -rf /opt/DDFacet/DDFacet/cbuild

RUN pip3 install -U pip setuptools wheel

RUN python3 -m pip install pybind11

RUN python3 -m pip install tensorflow==1.8.0

RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"

RUN cd /opt/DDFacet/ && python3 setup.py build && cd /

# Set MeqTrees Cattery path to installation directory

ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/

ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH



RUN python3 -m pip install numpy==1.17.0





RUN python3 -c "import Siamese"

RUN python3 -c "import bdsf"



# perform some basic tests

RUN DDF.py --help

RUN MakeMask.py --help

RUN MakeCatalog.py --help

RUN MakeModel.py --help

RUN MaskDicoModel.py --help

RUN ClusterCat.py --help



#####################################################################

## killMs

#####################################################################

ADD killMS /opt/killMS

RUN pip3 install /opt/killMS

RUN cd  /opt/killMS; python3 setup.py build



#####################################################################

## DDF-PIPELINE

#####################################################################

RUN pip3 install pyregion sshtunnel pymysql

RUN  cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline

RUN sed -e "s|INSTALLDIR|/opt|" /opt/ddf-pipeline/misc/DDF.sh > /opt/DDFacet/init.sh



RUN cd /opt && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git



ENV DDF_PIPELINE_CATALOGS='/opt/container-data/bootstrap-cats/'

ENV CONTAINER_NAME='ddfacet_ssd-py3_kms-lofar-release-4_0_16'
 
 No newline at end of file

ddf2/apt.sources.list

0 → 100644
+12 −0
Original line number Diff line number Diff line 
###### Ubuntu Main Repos

deb http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse

deb-src http://za.archive.ubuntu.com/ubuntu bionic main restricted universe multiverse

###### Ubuntu Update Repos

deb http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse

deb http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse

deb http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse

deb http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse

deb-src http://za.archive.ubuntu.com/ubuntu bionic-security main restricted universe multiverse

deb-src http://za.archive.ubuntu.com/ubuntu bionic-updates main restricted universe multiverse

deb-src http://za.archive.ubuntu.com/ubuntu bionic-proposed main restricted universe multiverse

deb-src http://za.archive.ubuntu.com/ubuntu bionic-backports main restricted universe multiverse

ddf2/build_docker_container.sh

+8 −2
Original line number Diff line number Diff line 
#!/bin/bash

set -e 

rm -rf DDFacet

rm -rf killMS



git clone https://github.com/cyriltasse/killMS.git

git checkout 0607798





git clone https://github.com/cyriltasse/DDFacet.git

git checkout 1059028

git checkout e98968d





docker build  . -t lofarit/ddf2_base3.10 
docker build  . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16