Commit c9468ede authored by Stefano Alberto Russo's avatar Stefano Alberto Russo
Browse files

Merge branch 'feature/ddf2_deployment_keys' into develop

parents e5ff5d8d 15a1414d

.gitignore

+1 −3
Original line number Original line Diff line number Diff line 


ddf2/keys
 
ddf2/DDFacet/

ddf2/killMS/

ddf2/Dockerfile

+61 −19
Original line number Original line Diff line number Diff line
@@ -65,21 +65,48 @@ RUN ln -s /usr/bin/g++-$GNUCOMPILER /usr/bin/cpp 
RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran

RUN ln -s /usr/bin/gfortran-$GNUCOMPILER /usr/bin/gfortran









#####################################################################

# Add deployment keys

#####################################################################



RUN mkdir -p /root/.ssh/ 

COPY keys/lofarit_ddf2_rsa /root/.ssh/lofarit_ddf2_rsa

COPY keys/lofarit_killms_rsa /root/.ssh/lofarit_killms_rsa

RUN chmod 600 /root/.ssh/lofarit_ddf2_rsa && chmod 600 /root/.ssh/lofarit_killms_rsa







#####################################################################

#####################################################################

## BUILD DDF FROM SOURCE

# Get DDF from private repo

#####################################################################

#####################################################################



# Add github key(s)

RUN ssh-keyscan github.com >> ~/.ssh/known_hosts



# Clone

RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/ && git clone git@github.com:cyriltasse/DDFacet'



# Checkout

RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_ddf2_rsa && cd /root/DDFacet && git pull && git checkout e98968d'







#####################################################################

# Build DDF from source

#####################################################################



RUN mkdir /opt/DDFacet



# Copy DDFacet and SkyModel into the image

# Copy DDFacet and SkyModel into the image

ADD DDFacet/DDFacet /opt/DDFacet/DDFacet

RUN cp -a /root/DDFacet/DDFacet /opt/DDFacet/DDFacet && \

ADD DDFacet/SkyModel /opt/DDFacet/SkyModel

    cp -a /root/DDFacet/SkyModel /opt/DDFacet/SkyModel && \

ADD DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in

    cp -a /root/DDFacet/MANIFEST.in /opt/DDFacet/MANIFEST.in && \

ADD DDFacet/setup.py /opt/DDFacet/setup.py

    cp -a /root/DDFacet/setup.py /opt/DDFacet/setup.py && \

ADD DDFacet/setup.cfg /opt/DDFacet/setup.cfg

    cp -a /root/DDFacet/setup.cfg /opt/DDFacet/setup.cfg && \

ADD DDFacet/README.rst /opt/DDFacet/README.rst

    cp -a /root/DDFacet/README.rst /opt/DDFacet/README.rst && \

ADD DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml

    cp -a /root/DDFacet/pyproject.toml /opt/DDFacet/pyproject.toml && \

ADD DDFacet/.git /opt/DDFacet/.git

    cp -a /root/DDFacet/.git /opt/DDFacet/.git && \

ADD DDFacet/.gitignore /opt/DDFacet/.gitignore

    cp -a /root/DDFacet/.gitignore /opt/DDFacet/.gitignore && \

ADD DDFacet/.gitmodules /opt/DDFacet/.gitmodules

    cp -a /root/DDFacet/.gitmodules /opt/DDFacet/.gitmodules









# Finally install DDFacet

# Finally install DDFacet
@@ -89,17 +116,17 @@ RUN python3 -m pip install pybind11 
RUN python3 -m pip install tensorflow==1.8.0

RUN python3 -m pip install tensorflow==1.8.0

RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"

RUN python3 -m pip install -U "/opt/DDFacet/[dft-support,moresane-support,testing-requirements,fits-beam-support]"

RUN cd /opt/DDFacet/ && python3 setup.py build && cd /

RUN cd /opt/DDFacet/ && python3 setup.py build && cd /



# Set MeqTrees Cattery path to installation directory

# Set MeqTrees Cattery path to installation directory

ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/

ENV MEQTREES_CATTERY_PATH /usr/local/lib/python3.6/dist-packages/Cattery/

ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH

ENV PYTHONPATH $MEQTREES_CATTERY_PATH:$PYTHONPATH





RUN python3 -m pip install numpy==1.17.0

RUN python3 -m pip install numpy==1.17.0







RUN python3 -c "import Siamese"

RUN python3 -c "import Siamese"

RUN python3 -c "import bdsf"

RUN python3 -c "import bdsf"





# perform some basic tests

# Perform some basic tests

RUN DDF.py --help

RUN DDF.py --help

RUN MakeMask.py --help

RUN MakeMask.py --help

RUN MakeCatalog.py --help

RUN MakeCatalog.py --help
@@ -107,15 +134,30 @@ RUN MakeModel.py --help 
RUN MaskDicoModel.py --help

RUN MaskDicoModel.py --help

RUN ClusterCat.py --help

RUN ClusterCat.py --help







#####################################################################

#####################################################################

## killMs

# Get and build killMs

#####################################################################

#####################################################################

ADD killMS /opt/killMS



# Clone

RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/ && git clone git@github.com:cyriltasse/killMS'



# Checkout

RUN ssh-agent bash -c 'ssh-add /root/.ssh/lofarit_killms_rsa && cd /opt/killMS && git pull && git checkout 0607798'



RUN pip3 install /opt/killMS

RUN pip3 install /opt/killMS

RUN cd  /opt/killMS; python3 setup.py build

RUN cd  /opt/killMS; python3 setup.py build







#####################################################################

# Remove deploymnet keys

#####################################################################



RUN rm -f /root/.ssh/lofarit_ddf2_rsa && rm -f /root/.ssh/lofarit_killms_rsa





#####################################################################

#####################################################################

## DDF-PIPELINE

# DDF-PIPELINE

#####################################################################

#####################################################################

RUN pip3 install pyregion sshtunnel pymysql

RUN pip3 install pyregion sshtunnel pymysql

RUN  cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline

RUN  cd /opt; git clone https://github.com/mhardcastle/ddf-pipeline

ddf2/build_docker_container.sh

+0 −11
Original line number Original line Diff line number Diff line 
#!/bin/bash

#!/bin/bash

set -e 

rm -rf DDFacet

rm -rf killMS



git clone https://github.com/cyriltasse/killMS.git

git checkout 0607798





git clone https://github.com/cyriltasse/DDFacet.git

git checkout e98968d







docker build  . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16
 
docker build  . -t lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16

ddf2/run_docker_container.sh

+1 −1
Original line number Original line Diff line number Diff line 
#!/bin/bash

#!/bin/bash

docker run --rm -v $PWD/data:/data -it lofarit/ddf2_base3.10 /bin/bash
 
docker run --rm -v $PWD/data:/data -it lofarit/ddfacet_ssd-py3_kms-lofar-release-4_0_16 /bin/bash