Commit 78c47609 authored by Giuliano Taffoni's avatar Giuliano Taffoni
Browse files

Initial commit

parents

base/Dockerfile

0 → 100644
+79 −0
Original line number Diff line number Diff line 
FROM morgan1971/lofar_metadesktop:1.0

MAINTAINER Giuliano Taffoni <giuliano.taffoni@inaf.it>

USER root

ENV CONTAINER_NAME='LofarBase'

# Set non-interactive

ENV DEBIAN_FRONTEND noninteractive

ENV GNUCOMPILER 7



ENV DEB_SETUP_DEPENDENCIES \

    dpkg-dev \

    g++-$GNUCOMPILER \

    gcc-$GNUCOMPILER \

    libc-dev \

    cmake \

    gfortran-$GNUCOMPILER \

    git \

    wget \

    subversion



ENV DEB_DEPENCENDIES \

            python3-pip \

            libfftw3-dev \

            python3-numpy \

            libfreetype6 \

            libfreetype6-dev \

            libpng-dev \

            pkg-config \

            python3-dev \

            libboost-all-dev \

            libcfitsio-dev \

            libhdf5-dev \

            wcslib-dev \

            libatlas-base-dev \

            liblapack-dev \

            python3-tk \

            libreadline6-dev \

            subversion \

            liblog4cplus-dev \

            libhdf5-dev \

            libncurses5-dev \

            libsofa1-dev \

            flex \

            bison \

            libbison-dev \

            # Reference image generation dependencies

            make \

            libgsl-dev \

            rsync \

            libopenblas-dev \

            libopenblas-base \

            emacs \

            imagemagick-6-common \

            saods9 \

            python3-numpy \

            libboost-python-dev libcfitsio-dev   cmake wget \

            build-essential libhdf5-serial-dev libarmadillo-dev \

            libboost-filesystem-dev libboost-system-dev libboost-date-time-dev \

            libboost-program-options-dev libboost-test-dev \

            libpng-dev pkg-config \

            libgtkmm-3.0-dev git libfftw3-dev \

                liblua5.3-dev \

                 pybind11-dev python3-pip ninja-build python3.6-venv virtualenv









RUN echo "*        -   memlock     unlimited" > /etc/security/limits.conf

RUN sudo apt-get -y install software-properties-common

RUN sudo add-apt-repository -y -s ppa:kernsuite/kern-6

RUN sudo apt-add-repository -y multiverse

RUN sudo apt-add-repository -y restricted

RUN sudo apt-get -y update

RUN sudo apt-get -y install apt-utils

RUN apt-get install -y $DEB_SETUP_DEPENDENCIES

RUN apt-get install -y $DEB_DEPENCENDIES

RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran

RUN apt-get -y clean

# Set user lofar

USER lofar

base/build-base.sh

0 → 100644
+9 −0
Original line number Diff line number Diff line 
#!/bin/bash



if [ 'xxx${VERSION}' == 'xxx' ]; then

   VERSION='latest'

fi







docker build -t morgan1971/lofar-base:${VERSION} -f Dockerfile .

ddf2/Dockerfile

0 → 100644
+114 −0
Original line number Diff line number Diff line 
FROM morgan1971/lofar-software:0.1.0

LABEL Description="This image is used to execute the LOFAR DDF 2.0"

LABEL maintainer="Giuliano Taffoni <giuliano.taffoni@inaf.it>"

ENV CONTAINER_NAME='Lofar DDF Software'

# Set non-interactive

ENV DEBIAN_FRONTEND noninteractive

ENV DDF_VERSION=5b95cdd

ENV PYBDSF_VERSION=v1.9.1



USER root

#####################################################################

# Add deployment keys

#####################################################################

RUN mkdir /pyenv-py3

COPY files/requirements3.txt /pyenv-py3

COPY files/requirements3_ddf.txt /pyenv-py3

RUN chown -R lofar.lofar /pyenv-py3







RUN mv /lofar_home_vanilla /home/lofar && mkdir -p /home/lofar/.ssh/

COPY keys/lofarit_ddf2_rsa /home/lofar/.ssh/lofarit_ddf2_rsa

COPY keys/lofarit_killms_rsa /home/lofar/.ssh/lofarit_killms_rsa

RUN  chmod 600 /home/lofar/.ssh/lofarit_ddf2_rsa && \

     chmod 600 /home/lofar/.ssh/lofarit_killms_rsa && \

     chown -R lofar.lofar /home/lofar/

RUN apt -y install python3-psycopg2

RUN python3 -m pip install -U pip setuptools wheel



USER lofar

#####################################################################

# Install requirements as local user

#####################################################################

RUN python3 -m pip install --upgrade pip

RUN pip3 install -r /pyenv-py3/requirements3.txt

RUN pip3 install -r /pyenv-py3/requirements3_ddf.txt



#####################################################################

# Install pybdfs 1.9.1

#####################################################################

RUN mkdir /tmp/pybdfs  && cd /tmp/pybdfs && \

    git clone https://github.com/lofar-astron/PyBDSF.git && \

    cd /tmp/pybdfs/PyBDSF && git checkout tags/${PYBDSF_VERSION} && \

    pip3 install /tmp/pybdfs/PyBDSF/ && rm -fr /tmp/pybdfs

LABEL it.inaf.lofar.software.pybdfs=1.9.1





#####################################################################

# Install DDF from private repo

#####################################################################

RUN ssh-keyscan github.com >> ~/.ssh/known_hosts && \

    ssh-agent bash -c 'ssh-add /home/lofar/.ssh/lofarit_ddf2_rsa && cd /tmp/ && git clone git@github.com:cyriltasse/DDFacet' && \

    ssh-agent bash -c 'ssh-add /home/lofar/.ssh/lofarit_ddf2_rsa && cd /tmp/DDFacet && git pull && git checkout $DDF_VERSION'

RUN python3 -c "import bdsf" && python3 -c "import siamese"

RUN pip3 install /tmp/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support,kms-support]

RUN rm -rf /tmp/DDFacet

#RUN DDF.py --help && MakeMask.py --help &&  MakeCatalog.py --help && MakeModel.py --help && \

#    MaskDicoModel.py --help && ClusterCat.py --help

LABEL it.inaf.lofar.software.ddf=5b95cdd



#####################################################################

#  Install killMs

#####################################################################

RUN ssh-agent bash -c 'ssh-add /home/lofar/.ssh/lofarit_killms_rsa && cd /tmp/ && git clone git@github.com:cyriltasse/killMS' && \

    ssh-agent bash -c 'ssh-add /home/lofar/.ssh/lofarit_killms_rsa && cd /tmp/killMS && git pull && git checkout InWeights'

RUN pip3 install /tmp/killMS && rm -fr /tmp/killMS

LABEL it.inaf.lofar.software.killMS=InWeights



#####################################################################

# DDF-PIPELINE

#####################################################################

RUN cd /pyenv-py3 &&  git clone https://github.com/mhardcastle/ddf-pipeline && \

    cd /pyenv-py3/ddf-pipeline && git checkout 8969b9f

RUN cd /pyenv-py3 && git clone https://www.ict.inaf.it/gitlab/lofarit/container-data.git



#####################################################################

# MH lotss-query

#####################################################################

RUN mkdir /pyenv-py3/lotss-query && cd /pyenv-py3/lotss-query && \

    git clone https://github.com/mhardcastle/lotss-query.git && \

    cp /pyenv-py3/lotss-query/lotss-query/surveys_db.py /pyenv-py3/ddf-pipeline/utils



RUN pip3 cache purge

#

# Final configurarations

#

USER root







RUN sudo apt-get -y clean

#RUN sudo ln -s /usr/share/lxde/wallpapers/lxde_blue.jpg /etc/alternatives/desktop-background

#

# PATHS and enviroments

#

RUN sed -e "s|INSTALLDIR|/pyenv-py3|" /pyenv-py3/ddf-pipeline/misc/DDF.sh > /pyenv-py3/DDF.sh

RUN echo "sh /pyenv-py3/DDF.sh" >> /home/lofar/.bashrc

ENV PATH /usr/local/bin:/home/lofar/.local/bin:$PATH

ENV LD_LIBRARY_PATH /usr/local/lib:$LD_LIBRARY_PATH

ENV DDF_PIPELINE_CATALOGS='/opt/container-data/bootstrap-cats/'

ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/

RUN echo "echo PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH" >> /home/lofar/.bashrc

RUN echo "echo PYTHONPATH /usr/local/lib/python3.6/site-packages:$PYTHONPATH" >> /home/lofar/.bashrc

RUN echo "echo PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH" >> /home/lofar/.bashrc

# Rename user home folder as a "vanilla" home folder

RUN mv /home/lofar /lofar_home_vanilla

RUN chmod -R ugo+r /lofar_home_vanilla && \

    chmod  ugo+x /lofar_home_vanilla/.local && \

    chmod  ugo+x /lofar_home_vanilla/.local/lib && \

    chmod  ugo+x  /lofar_home_vanilla/.local/lib/python3.6 && \

    chmod  ugo+x  /lofar_home_vanilla/.local/lib/python3.6/site-packages

USER lofar

LABEL Vendor="INAF"

LABEL Version="0.1.0"

ddf2/build-ddf.sh

0 → 100644
+9 −0
Original line number Diff line number Diff line 
#!/bin/bash



if [ 'xxx${VERSION}' == 'xxx' ]; then

   VERSION='latest'

fi







docker build -t morgan1971/lofar-ddf2:${VERSION} -f Dockerfile .

ddf2/files/requirements.ddf.txt

0 → 100644
+183 −0
Original line number Diff line number Diff line 
APLpy==2.0.3

absl-py==0.12.0

amqp==2.2.2

anyjson==0.3.3

appdirs==1.4.4

asn1crypto==0.24.0

astLib==0.11.6

astor==0.8.1

astro-kittens==1.4.5

astro-pyxis==1.7.3

astropy==4.1

astropy-healpix==0.6

astroquery==0.4.2

attrdict==2.0.1

attrs==17.4.0

backcall==0.2.0

backports.shutil-get-terminal-size==1.0.0

bcrypt==3.2.0

bdsf @ file:///root/PyBDSF

beautifulsoup4==4.9.3

bleach==1.5.0

Cerberus==1.3.4

certifi==2020.12.5

cffi==1.14.5

chardet==3.0.4

click==6.7

codex-africanus==0.1.8

colorama==0.3.7

configobj==5.0.6

configparser==5.0.2

cryptography==3.4.7

cupshelpers==1.0

cycler==0.10.0

Cython==0.29.23

dask==2021.3.0

dataclasses==0.8

DDFacet @ file:///opt/DDFacet

deap==1.3.1

decorator==4.4.2

defer==1.0.6

distro-info===0.18ubuntu0.18.04.1

emcee==3.0.2

ephem==3.7.7.1

funcsigs==1.0.2

future==0.18.2

gast==0.4.0

grpcio==1.38.0

html5lib==0.9999999

httplib2==0.9.2

hypercube==0.3.4

idna==2.6

imageio==2.9.0

importlib-metadata==4.2.0

ipdb==0.13.8

ipython==7.16.1

ipython-genutils==0.2.0

jedi==0.18.0

joblib==1.0.1

keyring==10.6.0

keyrings.alt==3.0

killMS @ file:///opt/killMS

kittens==0.1.2

kiwisolver==1.3.1

kombu==4.1.0

language-selector==0.1

launchpadlib==1.10.6

lazr.restfulclient==0.13.5

lazr.uri==1.0.3

llvmlite==0.36.0

losoto==2.2

lxml==4.6.3

macaroonbakery==1.1.3

Markdown==3.3.4

matplotlib==3.3.4

meqtrees-cattery==1.7.2

mimeparse==0.1.3

montblanc @ git+https://github.com/ska-sa/montblanc.git@547008faa46d5798f682d9d00597351a67f1915e

networkx==2.5.1

nose==1.3.7

numba==0.53.1

numexpr==2.7.3

numpy==1.17.0

oauth==1.0.1

olefile==0.45.1

pandas==1.1.5

paramiko==2.7.2

parso==0.8.2

pep517==0.10.0

pexpect==4.2.1

pickleshare==0.7.4

Pillow==8.2.0

pluggy==0.6.0

ply==3.11

Polygon3==3.0.9.1

prettytable==2.1.0

progressbar==2.5

prompt-toolkit==3.0.18

protobuf==3.17.1

psutil==5.4.2

psycopg2==2.7.4

ptyprocess==0.7.0

purr==1.5.2

py==1.5.2

py-cpuinfo==8.0.0

PyAVM==0.9.5

pybind11==2.6.2

pycairo==1.16.2

pycparser==2.20

pycrypto==2.6.1

pycups==1.9.73

pyephem==9.99

pyFFTW==0.12.0

pyfits==3.5

Pygments==2.2.0

pygobject==3.26.1

PyGreSQL==5.0.4

pylru==1.2.0

pymacaroons==0.13.0

pymoresane==0.3.0

PyMySQL==1.0.2

PyNaCl==1.1.2

pyparsing==2.2.0

pyregion==2.0

pyRFC3339==1.0

pytest==3.3.2

pytest-arraydiff==0.2

pytest-astropy==0.2.1

pytest-doctestplus==0.1.2

pytest-openfiles==0.2.0

pytest-remotedata==0.2.0

python-apt==1.6.5+ubuntu0.6

python-casacore==3.4.0

python-dateutil==2.8.1

python-debian==0.1.32

python-monetdb==11.19.3.2

python-qpid-proton==0.14.0

pytz==2018.3

pyvo==1.1

PyWavelets==1.1.1

pyxdg==0.25

PyYAML==3.12

reproject==0.7.1

requests==2.18.4

requests-unixsocket==0.1.5

ruamel.yaml==0.17.4

ruamel.yaml.clib==0.2.2

scikit-image==0.17.2

scikit-learn==0.24.2

scipy==1.3.3

SecretStorage==2.3.1

Shapely==1.7.1

SharedArray===b-3.1.0-2-gdc90bd2-n-

siamese==0.0.38

simplegeneric==0.8.1

simplejson==3.13.2

six==1.16.0

sklearn==0.0

soupsieve==2.2.1

sshtunnel==0.4.0

system-service==0.3

systemd-python==234

tables==3.6.1

tabulate==0.8.9

tensorboard==1.8.0

tensorflow==1.8.0

termcolor==1.1.0

threadpoolctl==2.1.0

tifffile==2020.9.3

toml==0.10.2

toolz==0.11.1

traitlets==4.3.2

typing-extensions==3.10.0.0

ubuntu-drivers-common==0.0.0

unattended-upgrades==0.1

urllib3==1.22

vine==1.1.4

wadllib==1.3.2

wcwidth==0.1.7

Werkzeug==2.0.1

xkit==0.0.0

xmlrunner==1.7.7

zipp==3.4.1

zope.interface==4.3.2