##########################################################################
# Pre-Facet Calibrator Calibration Pipeline v3.0 (04/09/2019)            #
#                                                                        #
# Calibrator part of the basic Pre-Facet calibration pipeline:           #
# - requires LOFAR software version  >= 3.1.0                            #
# - requires losoto software version >= 2.0.0                            #
# - expects shared filesystem, that all nodes can reach all files!       #
#   (E.g. a single workstation or compute cluster with shared filesystem #
#   doesn't work on multiple nodes on CEP3.)                             #
##########################################################################

##########################################
### parameters you will need to adjust. ##
##########################################
## information about the calibrator data
! cal_input_path           =  /data/lofar/input_data                                        ## specify the directory where your calibrator data is stored
! cal_input_pattern        =  *.MS                                                          ## regular expression pattern of all your calibrator files

## location of the software
! prefactor_directory      =  /opt/prefactor/                                            ## path to your prefactor copy
! losoto_directory         =  /opt/losoto                                                ## path to your local LoSoTo installation
! aoflagger                =  /opt/aoflagger/bin/aoflagger                               ## path to your aoflagger executable

##########################################
### parameters you may need to adjust   ##
##########################################

! refant                   =  'CS001HBA0'                                                         ## name of the station that will be used as a reference for the phase-plots, 'closest' will reference to the spatially closest unflagged antenna
! flag_baselines           =  []                                                                  ## NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] )
! process_baselines_cal    =  *&                                                                  ## performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
! filter_baselines         =  {{ process_baselines_cal }}                                         ## selects only this set of baselines to be processed. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
! do_smooth                =  False                                                               ## enable or disable baseline-based smoothing
! rfistrategy              =  HBAdefault.rfis                                                     ## strategy to be applied with the statistical flagger (AOFlagger)
! max2interpolate          =  30                                                                  ## amount of channels in which interpolation should be performed for deriving the bandpass
! ampRange                 =  [0, 0]                                                              ## range of median amplitudes accepted per station. Use [0, 0] to use typical ranges of [50, 225] for the current correlator or [0.0004, 0.0018] for the old one.
! skip_international       =  True                                                                ## skip fitting the bandpass for international stations (this avoids flagging them in many cases)
! raw_data                 =  False                                                               ## use autoweight, set to True in case you are using raw data
! propagatesolutions       =  True                                                                ## use already derived solutions as initial guess for the upcoming time slot
! flagunconverged          =  False                                                               ## flag solutions for solves that did not converge (if they were also detected to diverge)
! maxStddev                =  -1.0                                                                ## Maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes

# options for solution transfer (for non-trusted calibrator observations)
! solutions2transfer       =  {{ prefactor_directory }}/solutions/3C48.h5                         ## location of h5parm reference solutions to transfer
! antennas2transfer        =  [FUSPID].*                                                          ## regular expression of antennas which solutions should be transferred to the final solution set, if none of the trusted calibrators have been observed (by default: all international stations)

# demixing options (only used if demix step is added to the prep_cal_strategy variable)
! demix_sources            =  [CasA,CygA]                                                         ## choose sources to demix (provided as list)
! demix_target             =  ""                                                                  ## if given, the target source model (its patch in the SourceDB) is taken into account when solving
! demix_freqstep           =  16                                                                  ## number of channels to average when demixing.
! demix_timestep           =  10                                                                  ## number of time slots to average when demixing

# definitions for pipeline options -- do not change!
! default_flagging         =  flagbaseline,flagelev,flagamp                                       ## regular flagging after pre-processing by the observatory pipelines
! raw_flagging             =  flagedge,aoflag,{{ default_flagging }}                              ## full flagging (usually only necessary for raw data)
! 1st_order                =  ct,plotTEC,residuals                                                ## Do not change! Only cal_ion should be edited if needed
! 3rd_order                =  ct3,plotTEC,plotTEC3,residuals3                                     ## Do not change! Only cal_ion should be edited if needed
! demix                    =  demix,                                                              ## Do not change! Only demix_step should be edited if needed
! full_apply               =  ,apply_PA,apply_bandpass,apply_clock,apply_beam,apply_FR,apply_TEC  ## Do not change!
! none                     =                                                                      ## Do not change!

# pipeline options
! initial_flagging         =  {{ default_flagging }}                                              ## choose {{ raw_flagging }} if you process raw data
! demix_step               =  {{ none }}                                                          ## choose {{ demix }} if you want to demix
! cal_ion                  =  {{ 1st_order }},smooth                                              ## choose {{ 3rd_order }} if you want to include 3rd order ionospheric effects (may be useful for LBA < 35 MHz), add smooth if you want to use the median of the clock in time (suggested for HBA+LB)
! tables2export            =  clock                                                               ## comma-separated list of tables to export from the ionospheric calibration step (cal_ion)
! final_apply              =  {{ none }}                                                          ## choose {{ full_apply }} if you want to apply all extracted solutions to the calibrator field

##########################################
### parameters for pipeline performance ##
##########################################

! num_proc_per_node        =  input.output.max_per_node                                           ## number of processes to use per step per node (usually max_per_node from pipeline.cfg)
! num_proc_per_node_limit  =  4                                                                   ## number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration)
! max_dppp_threads         =  10                                                                  ## number of threads per process for NDPPP
! memoryperc               =  20                                                                  ## maximum of memory used for aoflagger in raw_flagging mode in percent
! min_length               =  50                                                                  ## minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read.
! overhead                 =  0.8                                                                 ## Only use this fraction of the available memory for deriving the amount of data to be concatenated.
! min_separation           =  30                                                                  ## minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING)
! max_separation_arcmin    =  1.0                                                                 ## maximum distance to the phase center for which a skymodel will be still accepted. Change with caution!

! error_tolerance          =  False                                                               ## set this to True if you want the pipeline run to continue if single bands fail

##########################################
### parameters you may want to adjust   ##
##########################################

## main directories
! lofar_directory          =  $LOFARROOT                                                          ## base directory of your LOFAR installation
! job_directory            =  input.output.job_directory                                          ## directory of the prefactor outputs
! working_directory        =  input.output.working_directory/input.output.job_name                ## specify the working_directory (intermediate data products)
! log_file                 =  input.output.log_file                                               ## location of the logfile
! mapfile_dir              =  input.output.mapfile_dir                                            ## specify mapfile directory

## script and plugin directories
! scripts                  =  {{ prefactor_directory }}/scripts
pipeline.pluginpath        =  {{ prefactor_directory }}/plugins

## skymodel directory
! calibrator_path_skymodel =  {{ prefactor_directory }}/skymodels
! A-team_skymodel          =  {{ calibrator_path_skymodel }}/Ateam_LBA_CC.skymodel

## result directories
! results_directory        =  {{ job_directory }}/results                                         ## location of the results
! inspection_directory     =  {{ results_directory }}/inspection                                  ## directory where the inspection plots will be stored
! cal_values_directory     =  {{ results_directory }}/cal_values                                  ## directory where the final h5parm solution set will be stored

## calibrator solutions
! cal_solutions            =  {{ cal_values_directory }}/cal_solutions.h5

## averaging for the calibrator data
! avg_timeresolution       =  4.                                                                  ## average to 4 sec/timeslot
! avg_freqresolution       =  48.82kHz                                                            ## average to 48.82 kHz/ch (= 4 ch/SB)
! bandpass_freqresolution  =  195.3125kHz                                                         ## resolution of the bandpass table is 195.3125kHz (= 1 ch/SB)

########################################################
##                                                    ##
##    BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE!  ##
##                                                    ##
########################################################

# which steps to run
pipeline.steps             =  [prep, PA, FR, bandpass, ion, finalize]


# pipeline substeps
pipeline.steps.prep        =  [createmap_cal, combine_data_map, check_Ateam_separation, mk_cal_values_dir, createmap_prepcal, createmap_instcal, create_ateam_model_map, make_sourcedb_ateam, expand_sourcedb_ateam, ndppp_prep_cal, combine_data_cal_map, ms_concat, ms_concat_map, expand_memory_map, aoflag, sky_cal, make_sourcedb, expand_sourcedb, expand_skymodel, calib_cal_parmmap, h5imp_cal_map, smooth_data, predict_cal]

pipeline.steps.PA          =  [calib_cal, h5imp_cal_PA, prepare_losoto_PA, process_losoto_PA, h5exp_cal_PA, apply_PA, apply_beam]
pipeline.steps.FR          =  [smooth_corrected, calib_cal,  h5imp_cal_FR, prepare_losoto_FR, process_losoto_FR, h5exp_cal_FR, apply_FR]
pipeline.steps.bandpass    =  [smooth_corrected, calib_cal2, h5imp_cal_bandpass, prepare_losoto_bandpass, prepare_losoto_bandpasstrans, process_losoto_bandpass, h5exp_cal_bandpass, transfer_solutions, apply_PA, apply_bandpass, apply_beam, apply_FR]
pipeline.steps.ion         =  [smooth_corrected, calib_cal2, h5imp_cal_ion, prepare_losoto_ion, process_losoto_ion, h5exp_cal_ion]
pipeline.steps.finalize    =  [h5parm_name {{ final_apply }}, make_summary]

###############################
## Mapping calibrator files  ##
###############################

# generate a mapfile of all the calibrator data
createmap_cal.control.kind                                  =   plugin
createmap_cal.control.type                                  =   createMapfile
createmap_cal.control.method                                =   mapfile_from_folder
createmap_cal.control.mapfile_dir                           =   {{ mapfile_dir }}
createmap_cal.control.filename                              =   createmap_cal.mapfile
createmap_cal.control.folder                                =   {{ cal_input_path }}
createmap_cal.control.pattern                               =   {{ cal_input_pattern }}

# combine all entries into one mapfile, for the sortmap script
combine_data_map.control.kind                               =   plugin
combine_data_map.control.type                               =   createMapfile
combine_data_map.control.method                             =   mapfile_all_to_one
combine_data_map.control.mapfile_dir                        =   {{ mapfile_dir }}
combine_data_map.control.filename                           =   combine_data_map.mapfile
combine_data_map.control.mapfile_in                         =   createmap_cal.output.mapfile

# warn for potential nearby A-Team sources
check_Ateam_separation.control.type                         =   pythonplugin
check_Ateam_separation.control.executable                   =   {{ scripts }}/check_Ateam_separation.py
check_Ateam_separation.control.mapfile_in                   =   combine_data_map.output.mapfile
check_Ateam_separation.control.inputkey                     =   MSfile
check_Ateam_separation.argument.min_separation              =   {{ min_separation }}
check_Ateam_separation.argument.outputimage                 =   {{ inspection_directory }}/A-Team_elevation_calibrator.png
check_Ateam_separation.argument.flags                       =   [MSfile]

#############################
## Prepare for demixing    ##
#############################
# generate a mapfile of the calibrator
createmap_prepcal.control.kind                              =   plugin
createmap_prepcal.control.type                              =   makeResultsMapfile
createmap_prepcal.control.mapfile_dir                       =   {{ mapfile_dir }}
createmap_prepcal.control.filename                          =   createmap_prepcal.mapfile
createmap_prepcal.control.mapfile_in                        =   createmap_cal.output.mapfile
createmap_prepcal.control.target_dir                        =   {{ job_directory }}
createmap_prepcal.control.make_target_dir                   =   False
createmap_prepcal.control.new_suffix                        =   .ndppp_prep_cal

# generate a mapfile for the instrument table of the calibrator
createmap_instcal.control.kind                              =   plugin
createmap_instcal.control.type                              =   changeMapfile
createmap_instcal.control.mapfile_in                        =   createmap_prepcal.output.mapfile
createmap_instcal.control.join_files                        =   instrument
createmap_instcal.control.newname                           =   createmap_instcal.mapfile

# create a mapfile with the A-Team skymodel, length = 1
create_ateam_model_map.control.kind                         =   plugin
create_ateam_model_map.control.type                         =   addListMapfile
create_ateam_model_map.control.hosts                        =   ['localhost']
create_ateam_model_map.control.files                        =   [ {{ A-team_skymodel }} ]
create_ateam_model_map.control.mapfile_dir                  =   {{ mapfile_dir }}
create_ateam_model_map.control.filename                     =   ateam_model_name.mapfile

# make sourcedbs from the A-Team skymodel, length = 1
make_sourcedb_ateam.control.kind                            =   recipe
make_sourcedb_ateam.control.type                            =   executable_args
make_sourcedb_ateam.control.executable                      =   {{ lofar_directory }}/bin/makesourcedb
make_sourcedb_ateam.control.error_tolerance                 =   {{ error_tolerance }}
make_sourcedb_ateam.control.args_format                     =   lofar
make_sourcedb_ateam.control.outputkey                       =   out
make_sourcedb_ateam.control.mapfile_in                      =   create_ateam_model_map.output.mapfile
make_sourcedb_ateam.control.inputkey                        =   in
make_sourcedb_ateam.argument.format                         =   <
make_sourcedb_ateam.argument.outtype                        =   blob

# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb_ateam.control.kind                          =   plugin
expand_sourcedb_ateam.control.type                          =   expandMapfile
expand_sourcedb_ateam.control.mapfile_in                    =   make_sourcedb_ateam.output.mapfile
expand_sourcedb_ateam.control.mapfile_to_match              =   createmap_cal.output.mapfile
expand_sourcedb_ateam.control.mapfile_dir                   =   {{ mapfile_dir }}
expand_sourcedb_ateam.control.filename                      =   expand_sourcedb_ateam.datamap


###############################
## Prepare calibrator        ##
###############################
# create the cal_values_directory if needed
mk_cal_values_dir.control.kind                              =   plugin
mk_cal_values_dir.control.type                              =   makeDirectory
mk_cal_values_dir.control.directory                         =   {{ cal_values_directory }}

# run NDPPP on the calibrator data
ndppp_prep_cal.control.type                                 =   dppp
ndppp_prep_cal.control.max_per_node                         =   {{ num_proc_per_node_limit }}
ndppp_prep_cal.control.error_tolerance                      =   {{ error_tolerance }}
ndppp_prep_cal.argument.numthreads                          =   {{ max_dppp_threads }}
ndppp_prep_cal.argument.msin                                =   createmap_cal.output.mapfile    # The input data.
ndppp_prep_cal.argument.msin.datacolumn                     =   DATA
ndppp_prep_cal.argument.msin.baseline                       =   {{ filter_baselines }}
ndppp_prep_cal.argument.msin.autoweight                     =   {{ raw_data }}
ndppp_prep_cal.argument.msout.datacolumn                    =   DATA
ndppp_prep_cal.argument.msout.writefullresflag              =   False
ndppp_prep_cal.argument.msout.overwrite                     =   True
ndppp_prep_cal.argument.msout.storagemanager                =   "Dysco"
ndppp_prep_cal.argument.msout.storagemanager.databitrate    =   0
ndppp_prep_cal.argument.steps                               =   [{{ initial_flagging }},{{ demix_step }}avg]
ndppp_prep_cal.argument.flagedge.type                       =   preflagger
ndppp_prep_cal.argument.flagedge.chan                       =   [0..nchan/32-1,31*nchan/32..nchan-1] # we are running on a single subband
ndppp_prep_cal.argument.aoflag.type                         =   aoflagger
ndppp_prep_cal.argument.aoflag.memoryperc                   =   {{ memoryperc }}
ndppp_prep_cal.argument.aoflag.keepstatistics               =   false
ndppp_prep_cal.argument.flagbaseline.type                   =   preflagger
ndppp_prep_cal.argument.flagbaseline.baseline               =   {{ flag_baselines }}
ndppp_prep_cal.argument.flagelev.type                       =   preflagger
ndppp_prep_cal.argument.flagelev.elevation                  =   0deg..20deg
ndppp_prep_cal.argument.flagamp.type                        =   preflagger
ndppp_prep_cal.argument.flagamp.amplmin                     =   1e-30
ndppp_prep_cal.argument.avg.type                            =   average
ndppp_prep_cal.argument.avg.timeresolution                  =   {{ avg_timeresolution }}
ndppp_prep_cal.argument.avg.freqresolution                  =   {{ avg_freqresolution }}
ndppp_prep_cal.argument.flagamp.type                        =   preflagger
ndppp_prep_cal.argument.flagamp.amplmin                     =   1e-30
ndppp_prep_cal.argument.demix.type                          =   demixer
ndppp_prep_cal.argument.demix.baseline                      =   {{ process_baselines_cal }}
ndppp_prep_cal.argument.demix.demixfreqstep                 =   {{ demix_freqstep }}
ndppp_prep_cal.argument.demix.demixtimestep                 =   {{ demix_timestep }}
ndppp_prep_cal.argument.demix.ignoretarget                  =   False
ndppp_prep_cal.argument.demix.targetsource                  =   {{ demix_target }}
ndppp_prep_cal.argument.demix.subtractsources               =   {{ demix_sources }}
ndppp_prep_cal.argument.demix.ntimechunk                    =   {{ max_dppp_threads }}
ndppp_prep_cal.argument.demix.skymodel                      =   expand_sourcedb_ateam.output.mapfile
ndppp_prep_cal.argument.demix.freqstep                      =   1
ndppp_prep_cal.argument.demix.timestep                      =   1
ndppp_prep_cal.argument.demix.instrumentmodel               =   createmap_instcal.output.mapfile

# combine all entries into one mapfile (just for the find_skymodel_cal_auto script)
combine_data_cal_map.control.kind                           =   plugin
combine_data_cal_map.control.type                           =   createMapfile
combine_data_cal_map.control.method                         =   mapfile_all_to_one
combine_data_cal_map.control.mapfile_dir                    =   {{ mapfile_dir }}
combine_data_cal_map.control.filename                       =   combine_data_cal_map.mapfile
combine_data_cal_map.control.mapfile_in                     =   ndppp_prep_cal.output.mapfile

# virtually concatenate calibrator subbands
ms_concat.control.type                                      =   pythonplugin
ms_concat.control.executable                                =   {{ scripts }}/concat_MS.py
ms_concat.control.error_tolerance                           =   {{ error_tolerance }}
ms_concat.argument.filename                                 =   concatmapfile.mapfile
ms_concat.argument.mapfile_dir                              =   {{ mapfile_dir }}
ms_concat.argument.min_length                               =   {{ min_length }}
ms_concat.argument.overhead                                 =   {{ overhead }}
ms_concat.argument.flags                                    =   [combine_data_cal_map.output.mapfile,outputkey]

# convert the output of ms_concat into usable mapfiles
ms_concat_map.control.kind                                  =   plugin
ms_concat_map.control.type                                  =   mapfilenamesFromMapfiles
ms_concat_map.control.mapfile_concatmap                     =   ms_concat.output.concatmapfile.mapfile

# convert the output of ms_concat_target into usable mapfiles
expand_memory_map.control.kind                              =   plugin
expand_memory_map.control.type                              =   expandMapfile
expand_memory_map.control.mapfile_in                        =   ms_concat.output.memory.mapfile
expand_memory_map.control.mapfile_to_match                  =   ms_concat_map.output.concatmap
expand_memory_map.control.mapfile_dir                       =   {{ mapfile_dir }}
expand_memory_map.control.filename                          =   expand_memory_map.mapfile

# run aoflagger on the concatenated data
aoflag.control.kind                                         =   recipe
aoflag.control.type                                         =   executable_args
aoflag.control.inplace                                      =   True
aoflag.control.executable                                   =   {{ aoflagger }}
aoflag.control.max_per_node                                 =   1
aoflag.control.error_tolerance                              =   {{ error_tolerance }}
aoflag.control.mapfiles_in                                  =   [ms_concat_map.output.concatmap,expand_memory_map.output.mapfile]
aoflag.control.inputkeys                                    =   [msin,memory]
aoflag.control.args_format                                  =   wsclean
aoflag.argument.strategy                                    =   {{ prefactor_directory }}/rfistrategies/{{ rfistrategy }}
aoflag.argument.flags                                       =   [-v,memory,-combine-spws,msin]

# find automatically the calibrator sky model
sky_cal.control.type                                        =   pythonplugin
sky_cal.control.executable                                  =   {{ scripts }}/find_skymodel_cal.py
sky_cal.control.error_tolerance                             =   {{ error_tolerance }}
sky_cal.argument.flags                                      =   [combine_data_cal_map.output.mapfile]
sky_cal.argument.DirSkymodelCal                             =   {{ calibrator_path_skymodel }}
sky_cal.argument.max_separation_arcmin                      =   {{ max_separation_arcmin }}

# make the sourcedb
make_sourcedb.control.kind                                  =   recipe
make_sourcedb.control.type                                  =   executable_args
make_sourcedb.control.executable                            =   {{ lofar_directory }}/bin/makesourcedb
make_sourcedb.control.error_tolerance                       =   {{ error_tolerance }}
make_sourcedb.control.args_format                           =   lofar
make_sourcedb.control.outputkey                             =   out
make_sourcedb.control.mapfile_in                            =   sky_cal.output.SkymodelCal.mapfile
make_sourcedb.control.inputkey                              =   in
make_sourcedb.argument.format                               =   <
make_sourcedb.argument.outtype                              =   blob

# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_sourcedb.control.kind                                =   plugin
expand_sourcedb.control.type                                =   expandMapfile
expand_sourcedb.control.mapfile_in                          =   make_sourcedb.output.mapfile
expand_sourcedb.control.mapfile_to_match                    =   ndppp_prep_cal.output.mapfile
expand_sourcedb.control.mapfile_dir                         =   {{ mapfile_dir }}
expand_sourcedb.control.filename                            =   expand_sourcedb.mapfile

# expand the sourcedb mapfile so that there is one entry for every file, length = nfiles
expand_skymodel.control.kind                                =   plugin
expand_skymodel.control.type                                =   expandMapfile
expand_skymodel.control.mapfile_in                          =   sky_cal.output.SkymodelName.mapfile
expand_skymodel.control.mapfile_to_match                    =   ndppp_prep_cal.output.mapfile
expand_skymodel.control.mapfile_dir                         =   {{ mapfile_dir }}
expand_skymodel.control.filename                            =   expand_skymodel.mapfile

# generate mapfile with the h5parm names to be used in the calib_cal steps
calib_cal_parmmap.control.kind                              =   plugin
calib_cal_parmmap.control.type                              =   createMapfile
calib_cal_parmmap.control.method                            =   add_suffix_to_file
calib_cal_parmmap.control.mapfile_in                        =   ndppp_prep_cal.output.mapfile
calib_cal_parmmap.control.add_suffix_to_file                =   /instrument.h5
calib_cal_parmmap.control.mapfile_dir                       =   {{ mapfile_dir }}
calib_cal_parmmap.control.filename                          =   calib_cal_h5parms.mapfile

# generate a mapfile with all files in a single entry
h5imp_cal_map.control.kind                                  =   plugin
h5imp_cal_map.control.type                                  =   compressMapfile
h5imp_cal_map.control.mapfile_in                            =   calib_cal_parmmap.output.mapfile
h5imp_cal_map.control.mapfile_dir                           =   {{ mapfile_dir }}
h5imp_cal_map.control.filename                              =   h5imp_cal_map.mapfile

# baseline-dependent smoothing
smooth_data.control.type                                    =   executable_args
smooth_data.control.inplace                                 =   True
smooth_data.control.max_per_node                            =   {{ num_proc_per_node }}
smooth_data.control.error_tolerance                         =   {{ error_tolerance }}
smooth_data.control.executable                              =   {{ scripts }}/BLsmooth.py
smooth_data.control.mapfile_in                              =   ndppp_prep_cal.output.mapfile
smooth_data.control.inputkey                                =   msin
smooth_data.argument.flags                                  =   [-S,{{ do_smooth }},-r,-i,DATA,-o,SMOOTHED_DATA,msin]

# baseline-dependent smoothing
smooth_corrected.control.type                               =   executable_args
smooth_corrected.control.inplace                            =   True
smooth_corrected.control.max_per_node                       =   {{ num_proc_per_node }}
smooth_corrected.control.error_tolerance                    =   {{ error_tolerance }}
smooth_corrected.control.executable                         =   {{ scripts }}/BLsmooth.py
smooth_corrected.control.mapfile_in                         =   ndppp_prep_cal.output.mapfile
smooth_corrected.control.inputkey                           =   msin
smooth_corrected.argument.flags                             =   [-S,{{ do_smooth }},-r,-i,CORRECTED_DATA,-o,SMOOTHED_DATA,msin]

# predict to save time
predict_cal.control.type                                    =   dppp
predict_cal.control.inplace                                 =   True
predict_cal.control.max_per_node                            =   {{ num_proc_per_node_limit }}
predict_cal.control.error_tolerance                         =   {{ error_tolerance }}
predict_cal.control.mapfiles_in                             =   [ndppp_prep_cal.output.mapfile,expand_sourcedb.output.mapfile,expand_skymodel.output.mapfile]
predict_cal.control.inputkeys                               =   [msfile,sourcedb,skymodel]
predict_cal.argument.msin                                   =   msfile
predict_cal.argument.numthreads                             =   {{ max_dppp_threads }}
predict_cal.argument.msin.datacolumn                        =   SMOOTHED_DATA
predict_cal.argument.msout.datacolumn                       =   MODEL_DATA
predict_cal.argument.msout.storagemanager                   =   "Dysco"
predict_cal.argument.msout.storagemanager.databitrate       =   0
predict_cal.argument.steps                                  =   [predict]
predict_cal.argument.predict.type                           =   predict
predict_cal.argument.predict.sourcedb                       =   sourcedb
predict_cal.argument.predict.sources                        =   skymodel
predict_cal.argument.predict.usebeammodel                   =   False # TODO: put to true for large sky
predict_cal.argument.predict.usechannelfreq                 =   False
predict_cal.argument.predict.beammode                       =   array_factor

# now run NDPPP on the averaged calibrator data (rotation+diagonal)
calib_cal.control.type                                      =   dppp
calib_cal.control.inplace                                   =   True
calib_cal.control.max_per_node                              =   {{ num_proc_per_node_limit }}
calib_cal.control.error_tolerance                           =   {{ error_tolerance }}
calib_cal.control.mapfiles_in                               =   [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile]
calib_cal.control.inputkeys                                 =   [msfile,h5parm]
calib_cal.argument.msin                                     =   msfile
calib_cal.argument.numthreads                               =   {{ max_dppp_threads }}
calib_cal.argument.msin.datacolumn                          =   SMOOTHED_DATA
calib_cal.argument.steps                                    =   [solve]
calib_cal.argument.solve.type                               =   ddecal
calib_cal.argument.solve.mode                               =   rotation+diagonal
calib_cal.argument.solve.h5parm                             =   h5parm
calib_cal.argument.solve.usemodelcolumn                     =   True
calib_cal.argument.solve.uvlambdamin                        =   300
calib_cal.argument.solve.maxiter                            =   50
calib_cal.argument.solve.nchan                              =   1
calib_cal.argument.solve.solint                             =   1
calib_cal.argument.solve.propagatesolutions                 =   {{ propagatesolutions }}
calib_cal.argument.solve.propagateconvergedonly             =   True
calib_cal.argument.solve.flagunconverged                    =   {{ flagunconverged }}
calib_cal.argument.solve.flagdivergedonly                   =   True
calib_cal.argument.solve.tolerance                          =   1e-3

# now run NDPPP on the averaged calibrator data (diagonal only)
calib_cal2.control.type                                     =   dppp
calib_cal2.control.inplace                                  =   True
calib_cal2.control.max_per_node                             =   {{ num_proc_per_node_limit }}
calib_cal2.control.error_tolerance                          =   {{ error_tolerance }}
calib_cal2.control.mapfiles_in                              =   [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile]
calib_cal2.control.inputkeys                                =   [msfile,h5parm]
calib_cal2.argument.msin                                    =   msfile
calib_cal2.argument.numthreads                              =   {{ max_dppp_threads }}
calib_cal2.argument.msin.datacolumn                         =   SMOOTHED_DATA
calib_cal2.argument.steps                                   =   [solve]
calib_cal2.argument.solve.type                              =   ddecal
calib_cal2.argument.solve.mode                              =   diagonal
calib_cal2.argument.solve.h5parm                            =   h5parm
calib_cal2.argument.solve.usemodelcolumn                    =   True
calib_cal2.argument.solve.uvlambdamin                       =   300
calib_cal2.argument.solve.maxiter                           =   50
calib_cal2.argument.solve.nchan                             =   1
calib_cal2.argument.solve.solint                            =   1
calib_cal2.argument.solve.propagatesolutions                =   {{ propagatesolutions }}
calib_cal2.argument.solve.propagateconvergedonly            =   True
calib_cal2.argument.solve.flagunconverged                   =   {{ flagunconverged }}
calib_cal2.argument.solve.flagdivergedonly                  =   True
calib_cal2.argument.solve.tolerance                         =   1e-3


###########################
## calibrate Pol. Align  ##
###########################
# collect all instrument tables into one h5parm
h5imp_cal_PA.control.kind                                   =   recipe
h5imp_cal_PA.control.type                                   =   executable_args
h5imp_cal_PA.control.executable                             =   {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_PA.control.error_tolerance                        =   {{ error_tolerance }}
h5imp_cal_PA.control.mapfile_in                             =   h5imp_cal_map.output.mapfile
h5imp_cal_PA.control.inputkey                               =   h5parm
h5imp_cal_PA.control.outputkey                              =   outh5parm
h5imp_cal_PA.argument.flags                                 =   [-q,-v,-c,h5parm]
h5imp_cal_PA.argument.outh5parm                             =   outh5parm

# create losoto v2 parset file
prepare_losoto_PA.control.kind                              =   plugin
prepare_losoto_PA.control.type                              =   makeLosotoParset
prepare_losoto_PA.control.steps                             =   [plotP3,plotPd,plotRot3,plotA3,bkp,align,plotAlign,residual,plotPr,plotPr2]
prepare_losoto_PA.control.filename                          =   {{ job_directory }}/losoto.parset
prepare_losoto_PA.control.global.ncpu                       =   {{ num_proc_per_node }}
prepare_losoto_PA.control.plotP3.operation                  =   PLOT
prepare_losoto_PA.control.plotP3.soltab                     =   sol000/phase000
prepare_losoto_PA.control.plotP3.axesInPlot                 =   [time,freq]
prepare_losoto_PA.control.plotP3.axisInTable                =   ant
prepare_losoto_PA.control.plotP3.plotFlag                   =   True
prepare_losoto_PA.control.plotP3.prefix                     =   {{ inspection_directory }}/polalign_ph_
prepare_losoto_PA.control.plotP3.refAnt                     =   {{ refant }}
prepare_losoto_PA.control.plotP3.minmax                     =   [-3.14,3.14]
prepare_losoto_PA.control.plotPd.operation                  =   PLOT
prepare_losoto_PA.control.plotPd.soltab                     =   sol000/phase000
prepare_losoto_PA.control.plotPd.axesInPlot                 =   [time,freq]
prepare_losoto_PA.control.plotPd.axisInTable                =   ant
prepare_losoto_PA.control.plotPd.axisDiff                   =   pol
prepare_losoto_PA.control.plotPd.plotFlag                   =   True
prepare_losoto_PA.control.plotPd.prefix                     =   {{ inspection_directory }}/polalign_ph_poldif
prepare_losoto_PA.control.plotPd.refAnt                     =   {{ refant }}
prepare_losoto_PA.control.plotPd.minmax                     =   [-3.14,3.14]
prepare_losoto_PA.control.plotRot3.operation                =   PLOT
prepare_losoto_PA.control.plotRot3.soltab                   =   sol000/rotation000
prepare_losoto_PA.control.plotRot3.axesInPlot               =   [time,freq]
prepare_losoto_PA.control.plotRot3.axisInTable              =   ant
prepare_losoto_PA.control.plotRot3.plotFlag                 =   True
prepare_losoto_PA.control.plotRot3.prefix                   =   {{ inspection_directory }}/polalign_rotangle
prepare_losoto_PA.control.plotRot3.refAnt                   =   {{ refant }}
prepare_losoto_PA.control.plotA3.operation                  =   PLOT
prepare_losoto_PA.control.plotA3.soltab                     =   sol000/amplitude000
prepare_losoto_PA.control.plotA3.axesInPlot                 =   [time,freq]
prepare_losoto_PA.control.plotA3.axisInTable                =   ant
prepare_losoto_PA.control.plotA3.plotFlag                   =   True
prepare_losoto_PA.control.plotA3.prefix                     =   {{ inspection_directory }}/polalign_amp_
prepare_losoto_PA.control.bkp.operation                     =   DUPLICATE
prepare_losoto_PA.control.bkp.soltab                        =   sol000/phase000
prepare_losoto_PA.control.bkp.soltabOut                     =   phaseOrig
prepare_losoto_PA.control.align.soltab                      =   sol000/phase000
prepare_losoto_PA.control.align.operation                   =   POLALIGN
prepare_losoto_PA.control.align.soltabOut                   =   polalign
prepare_losoto_PA.control.align.average                     =   True
prepare_losoto_PA.control.align.replace                     =   True
prepare_losoto_PA.control.align.fitOffset                   =   False
prepare_losoto_PA.control.align.refAnt                      =   {{ refant }}
prepare_losoto_PA.control.plotAlign.operation               =   PLOT
prepare_losoto_PA.control.plotAlign.soltab                  =   sol000/polalign
prepare_losoto_PA.control.plotAlign.axesInPlot              =   [time,freq]
prepare_losoto_PA.control.plotAlign.axisInTable             =   ant
prepare_losoto_PA.control.plotAlign.axisDiff                =   pol
prepare_losoto_PA.control.plotAlign.plotFlag                =   True
prepare_losoto_PA.control.plotAlign.prefix                  =   {{ inspection_directory }}/polalign
prepare_losoto_PA.control.plotAlign.refAnt                  =   {{ refant }}
prepare_losoto_PA.control.plotAlign.minmax                  =   [-3.14,3.14]
prepare_losoto_PA.control.residual.operation                =   RESIDUALS
prepare_losoto_PA.control.residual.soltab                   =   sol000/phase000
prepare_losoto_PA.control.residual.soltabsToSub             =   polalign
prepare_losoto_PA.control.plotPr.operation                  =   PLOT
prepare_losoto_PA.control.plotPr.soltab                     =   sol000/phase000
prepare_losoto_PA.control.plotPr.axesInPlot                 =   [time,freq]
prepare_losoto_PA.control.plotPr.axisInTable                =   ant
prepare_losoto_PA.control.plotPr.axisDiff                   =   pol
prepare_losoto_PA.control.plotPr.plotFlag                   =   True
prepare_losoto_PA.control.plotPr.prefix                     =   {{ inspection_directory }}/polalign_ph-res_poldif
prepare_losoto_PA.control.plotPr.refAnt                     =   {{ refant }}
prepare_losoto_PA.control.plotPr.minmax                     =   [-3.14,3.14]
prepare_losoto_PA.control.plotPr2.operation                 =   PLOT
prepare_losoto_PA.control.plotPr2.soltab                    =   sol000/phase000
prepare_losoto_PA.control.plotPr2.axesInPlot                =   [time,freq]
prepare_losoto_PA.control.plotPr2.axisInTable               =   ant
prepare_losoto_PA.control.plotPr2.axisInCol                 =   pol
prepare_losoto_PA.control.plotPr2.plotFlag                  =   True
prepare_losoto_PA.control.plotPr2.prefix                    =   {{ inspection_directory }}/polalign_ph-res_
prepare_losoto_PA.control.plotPr2.refAnt                    =   {{ refant }}
prepare_losoto_PA.control.plotPr2.minmax                    =   [-3.14,3.14]

# do the processing on the LoSoTo file
process_losoto_PA.control.kind                              =   recipe
process_losoto_PA.control.type                              =   executable_args
process_losoto_PA.control.inplace                           =   True
process_losoto_PA.control.executable                        =   {{ losoto_directory }}/bin/losoto
process_losoto_PA.control.max_per_node                      =   {{ num_proc_per_node }}
process_losoto_PA.control.mapfile_in                        =   h5imp_cal_PA.output.mapfile
process_losoto_PA.control.inputkey                          =   h5in
process_losoto_PA.argument.flags                            =   [-v,h5in,{{ job_directory }}/losoto.parset]

# output the final soltab into an external h5parm
h5exp_cal_PA.control.kind                                   =   recipe
h5exp_cal_PA.control.type                                   =   executable_args
h5exp_cal_PA.control.inplace                                =   True
h5exp_cal_PA.control.executable                             =   {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_PA.control.error_tolerance                        =   {{ error_tolerance }}
h5exp_cal_PA.control.mapfile_in                             =   h5imp_cal_PA.output.mapfile
h5exp_cal_PA.control.inputkey                               =   h5in
h5exp_cal_PA.argument.flags                                 =   [-q,-v,h5in]
h5exp_cal_PA.argument.insoltab                              =   polalign
h5exp_cal_PA.argument.outh5parm                             =   {{ cal_solutions }}
h5exp_cal_PA.argument.outsolset                             =   calibrator

################################
## calibrate Farady Rotation  ##
################################
# collect all instrument tables into one h5parm
h5imp_cal_FR.control.kind                                   =   recipe
h5imp_cal_FR.control.type                                   =   executable_args
h5imp_cal_FR.control.executable                             =   {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_FR.control.error_tolerance                        =   {{ error_tolerance }}
h5imp_cal_FR.control.mapfile_in                             =   h5imp_cal_map.output.mapfile
h5imp_cal_FR.control.inputkey                               =   h5parm
h5imp_cal_FR.control.outputkey                              =   outh5parm
h5imp_cal_FR.argument.flags                                 =   [-q,-v,-c,h5parm]
h5imp_cal_FR.argument.outh5parm                             =   outh5parm

# create losoto v2 parset file
prepare_losoto_FR.control.kind                              =   plugin
prepare_losoto_FR.control.type                              =   makeLosotoParset
prepare_losoto_FR.control.steps                             =   [plotP3,plotPd,plotRot3,plotA3,faraday,plotFR,residual,plotPr,plotPr2]
prepare_losoto_FR.control.filename                          =   {{ job_directory }}/losoto.parset
prepare_losoto_FR.control.global.ncpu                       =   {{ num_proc_per_node }}
prepare_losoto_FR.control.plotP3.operation                  =   PLOT
prepare_losoto_FR.control.plotP3.soltab                     =   sol000/phase000
prepare_losoto_FR.control.plotP3.axesInPlot                 =   [time,freq]
prepare_losoto_FR.control.plotP3.axisInTable                =   ant
prepare_losoto_FR.control.plotP3.plotFlag                   =   True
prepare_losoto_FR.control.plotP3.prefix                     =   {{ inspection_directory }}/fr_ph_
prepare_losoto_FR.control.plotP3.refAnt                     =   {{ refant }}
prepare_losoto_FR.control.plotP3.minmax                     =   [-3.14,3.14]
prepare_losoto_FR.control.plotPd.operation                  =   PLOT
prepare_losoto_FR.control.plotPd.soltab                     =   sol000/phase000
prepare_losoto_FR.control.plotPd.axesInPlot                 =   [time,freq]
prepare_losoto_FR.control.plotPd.axisInTable                =   ant
prepare_losoto_FR.control.plotPd.axisDiff                   =   pol
prepare_losoto_FR.control.plotPd.plotFlag                   =   True
prepare_losoto_FR.control.plotPd.prefix                     =   {{ inspection_directory }}/fr_ph_poldif
prepare_losoto_FR.control.plotPd.refAnt                     =   {{ refant }}
prepare_losoto_FR.control.plotPd.minmax                     =   [-3.14,3.14]
prepare_losoto_FR.control.plotRot3.operation                =   PLOT
prepare_losoto_FR.control.plotRot3.soltab                   =   sol000/rotation000
prepare_losoto_FR.control.plotRot3.axesInPlot               =   [time,freq]
prepare_losoto_FR.control.plotRot3.axisInTable              =   ant
prepare_losoto_FR.control.plotRot3.plotFlag                 =   True
prepare_losoto_FR.control.plotRot3.prefix                   =   {{ inspection_directory }}/fr_rotangle
prepare_losoto_FR.control.plotRot3.refAnt                   =   {{ refant }}
prepare_losoto_FR.control.plotA3.operation                  =   PLOT
prepare_losoto_FR.control.plotA3.soltab                     =   sol000/amplitude000
prepare_losoto_FR.control.plotA3.axesInPlot                 =   [time,freq]
prepare_losoto_FR.control.plotA3.axisInTable                =   ant
prepare_losoto_FR.control.plotA3.plotFlag                   =   True
prepare_losoto_FR.control.plotA3.prefix                     =   {{ inspection_directory }}/fr_amp_
prepare_losoto_FR.control.faraday.operation                 =   FARADAY
prepare_losoto_FR.control.faraday.soltab                    =   sol000/rotation000
prepare_losoto_FR.control.faraday.soltabOut                 =   faraday
prepare_losoto_FR.control.faraday.refAnt                    =   {{ refant }}
prepare_losoto_FR.control.faraday.maxResidual               =   1.
prepare_losoto_FR.control.plotFR.operation                  =   PLOT
prepare_losoto_FR.control.plotFR.soltab                     =   sol000/faraday
prepare_losoto_FR.control.plotFR.axesInPlot                 =   [time]
prepare_losoto_FR.control.plotFR.axisInTable                =   ant
prepare_losoto_FR.control.plotFR.prefix                     =   {{ inspection_directory }}/fr
prepare_losoto_FR.control.residual.operation                =   RESIDUALS
prepare_losoto_FR.control.residual.soltab                   =   sol000/phase000
prepare_losoto_FR.control.residual.soltabsToSub             =   faraday
prepare_losoto_FR.control.plotPr.operation                  =   PLOT
prepare_losoto_FR.control.plotPr.soltab                     =   sol000/phase000
prepare_losoto_FR.control.plotPr.axesInPlot                 =   [time,freq]
prepare_losoto_FR.control.plotPr.axisInTable                =   ant
prepare_losoto_FR.control.plotPr.axisDiff                   =   pol
prepare_losoto_FR.control.plotPr.plotFlag                   =   True
prepare_losoto_FR.control.plotPr.prefix                     =   {{ inspection_directory }}/fr_ph-res_poldif
prepare_losoto_FR.control.plotPr.refAnt                     =   {{ refant }}
prepare_losoto_FR.control.plotPr.minmax                     =   [-3.14,3.14]
prepare_losoto_FR.control.plotPr2.operation                 =   PLOT
prepare_losoto_FR.control.plotPr2.soltab                    =   sol000/phase000
prepare_losoto_FR.control.plotPr2.axesInPlot                =   [time,freq]
prepare_losoto_FR.control.plotPr2.axisInTable               =   ant
prepare_losoto_FR.control.plotPr2.axisInCol                 =   pol
prepare_losoto_FR.control.plotPr2.plotFlag                  =   True
prepare_losoto_FR.control.plotPr2.prefix                    =   {{ inspection_directory }}/fr_ph-res_
prepare_losoto_FR.control.plotPr2.refAnt                    =   {{ refant }}
prepare_losoto_FR.control.plotPr2.minmax                    =   [-3.14,3.14]

# do the processing on the LoSoTo file
process_losoto_FR.control.kind                              =   recipe
process_losoto_FR.control.type                              =   executable_args
process_losoto_FR.control.inplace                           =   True
process_losoto_FR.control.executable                        =   {{ losoto_directory }}/bin/losoto
process_losoto_FR.control.max_per_node                      =   {{ num_proc_per_node }}
process_losoto_FR.control.mapfile_in                        =   h5imp_cal_FR.output.mapfile
process_losoto_FR.control.inputkey                          =   h5in
process_losoto_FR.argument.flags                            =   [-v,h5in,{{ job_directory }}/losoto.parset]

# output the final soltab into an external h5parm
h5exp_cal_FR.control.kind                                   =   recipe
h5exp_cal_FR.control.type                                   =   executable_args
h5exp_cal_FR.control.inplace                                =   True
h5exp_cal_FR.control.executable                             =   {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_FR.control.error_tolerance                        =   {{ error_tolerance }}
h5exp_cal_FR.control.mapfile_in                             =   h5imp_cal_FR.output.mapfile
h5exp_cal_FR.control.inputkey                               =   h5in
h5exp_cal_FR.argument.flags                                 =   [-q,-v,h5in]
h5exp_cal_FR.argument.insoltab                              =   faraday
h5exp_cal_FR.argument.outh5parm                             =   {{ cal_solutions }}
h5exp_cal_FR.argument.outsolset                             =   calibrator

################################
## calibrate bandpass         ##
################################
# collect all instrument tables into one h5parm
h5imp_cal_bandpass.control.kind                             =   recipe
h5imp_cal_bandpass.control.type                             =   executable_args
h5imp_cal_bandpass.control.executable                       =   {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_bandpass.control.error_tolerance                  =   {{ error_tolerance }}
h5imp_cal_bandpass.control.mapfile_in                       =   h5imp_cal_map.output.mapfile
h5imp_cal_bandpass.control.inputkey                         =   h5parm
h5imp_cal_bandpass.control.outputkey                        =   outh5parm
h5imp_cal_bandpass.argument.flags                           =   [-q,-v,-c,h5parm]
h5imp_cal_bandpass.argument.outh5parm                       =   outh5parm

# create losoto v2 parset file
prepare_losoto_bandpass.control.kind                        =   plugin
prepare_losoto_bandpass.control.type                        =   makeLosotoParset
prepare_losoto_bandpass.control.steps                       =   [duplicateAbkp,plotA1,flag,flagbp,flagextend,plotA2,merge,smooth,bandpass,interp,smoothb,plotB1,plotB2,plotB3]
prepare_losoto_bandpass.control.filename                    =   {{ job_directory }}/losoto.parset
prepare_losoto_bandpass.control.global.ncpu                 =   {{ num_proc_per_node }}
prepare_losoto_bandpass.control.duplicateAbkp.operation     =   DUPLICATE
prepare_losoto_bandpass.control.duplicateAbkp.soltab        =   sol000/amplitude000
prepare_losoto_bandpass.control.duplicateAbkp.soltabOut     =   amplitudeOrig000
prepare_losoto_bandpass.control.plotA1.operation            =   PLOT
prepare_losoto_bandpass.control.plotA1.soltab               =   sol000/amplitude000
prepare_losoto_bandpass.control.plotA1.axesInPlot           =   [time,freq]
prepare_losoto_bandpass.control.plotA1.axisInTable          =   ant
prepare_losoto_bandpass.control.plotA1.plotFlag             =   True
prepare_losoto_bandpass.control.plotA1.prefix               =   {{ inspection_directory }}/ampBFlag_
prepare_losoto_bandpass.control.flag.operation              =   FLAG
prepare_losoto_bandpass.control.flag.soltab                 =   sol000/amplitude000
prepare_losoto_bandpass.control.flag.axesToFlag             =   [time,freq]
prepare_losoto_bandpass.control.flag.order                  =   [100,40]
prepare_losoto_bandpass.control.flag.maxCycles              =   1
prepare_losoto_bandpass.control.flag.maxRms                 =   5
prepare_losoto_bandpass.control.flag.replace                =   False
prepare_losoto_bandpass.control.flag.preFlagZeros           =   False
prepare_losoto_bandpass.control.flag.mode                   =   smooth
prepare_losoto_bandpass.control.flagbp.operation            =   FLAGSTATION
prepare_losoto_bandpass.control.flagbp.soltab               =   sol000/amplitude000
prepare_losoto_bandpass.control.flagbp.mode                 =   bandpass
prepare_losoto_bandpass.control.flagbp.ampRange             =   {{ ampRange }}
prepare_losoto_bandpass.control.flagbp.skipInternational    =   {{ skip_international }}
prepare_losoto_bandpass.control.flagextend.operation        =   FLAGEXTEND
prepare_losoto_bandpass.control.flagextend.soltab           =   sol000/amplitude000
prepare_losoto_bandpass.control.flagextend.axesToExt        =   [time,freq]
prepare_losoto_bandpass.control.flagextend.size             =   [200,80]
prepare_losoto_bandpass.control.flagextend.percent          =   50
prepare_losoto_bandpass.control.flagextend.maxCycles        =   2
prepare_losoto_bandpass.control.plotA2.operation            =   PLOT
prepare_losoto_bandpass.control.plotA2.soltab               =   sol000/amplitude000
prepare_losoto_bandpass.control.plotA2.axesInPlot           =   [time,freq]
prepare_losoto_bandpass.control.plotA2.axisInTable          =   ant
prepare_losoto_bandpass.control.plotA2.plotFlag             =   True
prepare_losoto_bandpass.control.plotA2.prefix               =   {{ inspection_directory }}/ampAFlag_
prepare_losoto_bandpass.control.merge.operation             =   REWEIGHT
prepare_losoto_bandpass.control.merge.mode                  =   copy
prepare_losoto_bandpass.control.merge.soltab                =   sol000/phase000
prepare_losoto_bandpass.control.merge.soltabImport          =   amplitude000
prepare_losoto_bandpass.control.smooth.operation            =   SMOOTH
prepare_losoto_bandpass.control.smooth.soltab               =   sol000/amplitude000
prepare_losoto_bandpass.control.smooth.pol                  =   XX,YY
prepare_losoto_bandpass.control.smooth.axesToSmooth         =   [time]
prepare_losoto_bandpass.control.smooth.mode                 =   median
prepare_losoto_bandpass.control.smooth.log                  =   True
prepare_losoto_bandpass.control.smooth.replace              =   False
prepare_losoto_bandpass.control.bandpass.operation          =   SMOOTH
prepare_losoto_bandpass.control.bandpass.soltab             =   sol000/amplitude000
prepare_losoto_bandpass.control.bandpass.pol                =   XX,YY
prepare_losoto_bandpass.control.bandpass.axesToSmooth       =   [freq]
prepare_losoto_bandpass.control.bandpass.mode               =   savitzky-golay
prepare_losoto_bandpass.control.bandpass.size               =   [17]
prepare_losoto_bandpass.control.bandpass.degree             =   2
prepare_losoto_bandpass.control.bandpass.log                =   True
prepare_losoto_bandpass.control.interp.operation            =   INTERPOLATE
prepare_losoto_bandpass.control.interp.soltab               =   sol000/amplitude000
prepare_losoto_bandpass.control.interp.outSoltab            =   bandpass
prepare_losoto_bandpass.control.interp.axisToRegrid         =   freq
prepare_losoto_bandpass.control.interp.maxFlaggedWidth      =   {{ max2interpolate }}
prepare_losoto_bandpass.control.interp.newDelta             =   {{ bandpass_freqresolution }}
prepare_losoto_bandpass.control.interp.delta                =   {{ avg_freqresolution }}
prepare_losoto_bandpass.control.interp.log                  =   True
prepare_losoto_bandpass.control.smoothb.operation           =   SMOOTH
prepare_losoto_bandpass.control.smoothb.soltab              =   sol000/bandpass
prepare_losoto_bandpass.control.smoothb.pol                 =   XX,YY
prepare_losoto_bandpass.control.smoothb.axesToSmooth        =   [time]
prepare_losoto_bandpass.control.smoothb.mode                =   median
prepare_losoto_bandpass.control.smoothb.log                 =   True
prepare_losoto_bandpass.control.smoothb.replace             =   True
prepare_losoto_bandpass.control.plotB1.operation            =   PLOT
prepare_losoto_bandpass.control.plotB1.soltab               =   sol000/bandpass
prepare_losoto_bandpass.control.plotB1.axesInPlot           =   [time,freq]
prepare_losoto_bandpass.control.plotB1.axisInTable          =   ant
prepare_losoto_bandpass.control.plotB1.plotFlag             =   True
prepare_losoto_bandpass.control.plotB1.prefix               =   {{ inspection_directory }}/bandpass_
prepare_losoto_bandpass.control.plotB2.operation            =   PLOT
prepare_losoto_bandpass.control.plotB2.soltab               =   sol000/bandpass
prepare_losoto_bandpass.control.plotB2.axesInPlot           =   freq
prepare_losoto_bandpass.control.plotB2.axisInTable          =   ant
prepare_losoto_bandpass.control.plotB2.axisInCol            =   pol
prepare_losoto_bandpass.control.plotB2.plotFlag             =   True
prepare_losoto_bandpass.control.plotB2.prefix               =   {{ inspection_directory }}/bandpass_
prepare_losoto_bandpass.control.plotB2.time.minmaxstep      =   [0,1e20,500000]
prepare_losoto_bandpass.control.plotB3.operation            =   PLOT
prepare_losoto_bandpass.control.plotB3.soltab               =   sol000/bandpass
prepare_losoto_bandpass.control.plotB3.axesInPlot           =   freq
prepare_losoto_bandpass.control.plotB3.axisInCol            =   ant
prepare_losoto_bandpass.control.plotB3.plotFlag             =   True
prepare_losoto_bandpass.control.plotB3.prefix               =   {{ inspection_directory }}/bandpass_
prepare_losoto_bandpass.control.plotB3.time.minmaxstep      =   [0,1e20,500000]

# create losoto v2 parset file
prepare_losoto_bandpasstrans.control.kind                   =   plugin
prepare_losoto_bandpasstrans.control.type                   =   makeLosotoParset
prepare_losoto_bandpasstrans.control.steps                  =   [plotB1,plotB2,plotB3]
prepare_losoto_bandpasstrans.control.filename               =   {{ job_directory }}/losoto_transfer.parset
prepare_losoto_bandpasstrans.control.global.ncpu            =   {{ num_proc_per_node }}
prepare_losoto_bandpasstrans.control.plotB1.operation       =   PLOT
prepare_losoto_bandpasstrans.control.plotB1.soltab          =   calibrator/bandpass
prepare_losoto_bandpasstrans.control.plotB1.axesInPlot      =   [time,freq]
prepare_losoto_bandpasstrans.control.plotB1.axisInTable     =   ant
prepare_losoto_bandpasstrans.control.plotB1.plotFlag        =   True
prepare_losoto_bandpasstrans.control.plotB1.prefix          =   {{ inspection_directory }}/bandpass_transfer_
prepare_losoto_bandpasstrans.control.plotB2.operation       =   PLOT
prepare_losoto_bandpasstrans.control.plotB2.soltab          =   calibrator/bandpass
prepare_losoto_bandpasstrans.control.plotB2.axesInPlot      =   freq
prepare_losoto_bandpasstrans.control.plotB2.axisInTable     =   ant
prepare_losoto_bandpasstrans.control.plotB2.axisInCol       =   pol
prepare_losoto_bandpasstrans.control.plotB2.plotFlag        =   True
prepare_losoto_bandpasstrans.control.plotB2.prefix          =   {{ inspection_directory }}/bandpass_transfer_
prepare_losoto_bandpasstrans.control.plotB2.time.minmaxstep =   [0,1e20,500000]
prepare_losoto_bandpasstrans.control.plotB3.operation       =   PLOT
prepare_losoto_bandpasstrans.control.plotB3.soltab          =   calibrator/bandpass
prepare_losoto_bandpasstrans.control.plotB3.axesInPlot      =   freq
prepare_losoto_bandpasstrans.control.plotB3.axisInCol       =   ant
prepare_losoto_bandpasstrans.control.plotB3.plotFlag        =   True
prepare_losoto_bandpasstrans.control.plotB3.prefix          =   {{ inspection_directory }}/bandpass_transfer_
prepare_losoto_bandpasstrans.control.plotB3.time.minmaxstep =   [0,1e20,500000]

# do the processing on the LoSoTo file
process_losoto_bandpass.control.kind                        =   recipe
process_losoto_bandpass.control.type                        =   executable_args
process_losoto_bandpass.control.inplace                     =   True
process_losoto_bandpass.control.executable                  =   {{ losoto_directory }}/bin/losoto
process_losoto_bandpass.control.max_per_node                =   {{ num_proc_per_node }}
process_losoto_bandpass.control.mapfile_in                  =   h5imp_cal_bandpass.output.mapfile
process_losoto_bandpass.control.inputkey                    =   h5in
process_losoto_bandpass.argument.flags                      =   [-v,h5in,{{ job_directory }}/losoto.parset]

# output the final soltab into an external h5parm
h5exp_cal_bandpass.control.kind                             =   recipe
h5exp_cal_bandpass.control.type                             =   executable_args
h5exp_cal_bandpass.control.inplace                          =   True
h5exp_cal_bandpass.control.executable                       =   {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_bandpass.control.error_tolerance                  =   {{ error_tolerance }}
h5exp_cal_bandpass.control.mapfile_in                       =   h5imp_cal_bandpass.output.mapfile
h5exp_cal_bandpass.control.inputkey                         =   h5in
h5exp_cal_bandpass.argument.flags                           =   [-q,-v,h5in]
h5exp_cal_bandpass.argument.insoltab                        =   bandpass
h5exp_cal_bandpass.argument.outh5parm                       =   {{ cal_solutions }}
h5exp_cal_bandpass.argument.outsolset                       =   calibrator

# transfer solutions if necessary
transfer_solutions.control.type                             =   pythonplugin
transfer_solutions.control.executable                       =   {{ scripts }}/transfer_solutions.py
transfer_solutions.control.error_tolerance                  =   {{ error_tolerance }}
transfer_solutions.control.skip_infile                      =   True
transfer_solutions.control.mapfile_in                       =   sky_cal.output.SkymodelName.mapfile
transfer_solutions.argument.flags                           =   [{{ cal_solutions }}, {{ solutions2transfer }}]
transfer_solutions.argument.insolset                        =   calibrator
transfer_solutions.argument.outsolset                       =   calibrator
transfer_solutions.argument.insoltab                        =   bandpass
transfer_solutions.argument.outsoltab                       =   bandpass
transfer_solutions.argument.antenna                         =   {{ antennas2transfer }}
transfer_solutions.argument.parset                          =   {{ job_directory }}/losoto_transfer.parset

################################
## calibrate ionosphere       ##
################################
# collect all instrument tables into one h5parm
h5imp_cal_ion.control.kind                                  =   recipe
h5imp_cal_ion.control.type                                  =   executable_args
h5imp_cal_ion.control.executable                            =   {{ losoto_directory }}/bin/H5parm_collector.py
h5imp_cal_ion.control.error_tolerance                       =   {{ error_tolerance }}
h5imp_cal_ion.control.mapfile_in                            =   h5imp_cal_map.output.mapfile
h5imp_cal_ion.control.inputkey                              =   h5parm
h5imp_cal_ion.control.outputkey                             =   outh5parm
h5imp_cal_ion.argument.flags                                =   [-q,-v,-c,h5parm]
h5imp_cal_ion.argument.outh5parm                            =   outh5parm

# create losoto v2 parset file
prepare_losoto_ion.control.kind                             =   plugin
prepare_losoto_ion.control.type                             =   makeLosotoParset
prepare_losoto_ion.control.steps                            =   [plotA1,flag,flagextend,plotA2,merge,plotP3,plotPd,duplicatePbkp,{{ cal_ion }},plotClock,plotPr,plotPr3,flagstation]
prepare_losoto_ion.control.filename                         =   {{ job_directory }}/losoto.parset
prepare_losoto_ion.control.global.ncpu                      =   {{ num_proc_per_node }}
prepare_losoto_ion.control.plotA1.operation                 =   PLOT
prepare_losoto_ion.control.plotA1.soltab                    =   sol000/amplitude000
prepare_losoto_ion.control.plotA1.axesInPlot                =   [time,freq]
prepare_losoto_ion.control.plotA1.axisInTable               =   ant
prepare_losoto_ion.control.plotA1.plotFlag                  =   True
prepare_losoto_ion.control.plotA1.prefix                    =   {{ inspection_directory }}/ion_ampBFlag_
prepare_losoto_ion.control.flag.operation                   =   FLAG
prepare_losoto_ion.control.flag.soltab                      =   sol000/amplitude000
prepare_losoto_ion.control.flag.axesToFlag                  =   [time,freq]
prepare_losoto_ion.control.flag.order                       =   [100,40]
prepare_losoto_ion.control.flag.maxCycles                   =   1
prepare_losoto_ion.control.flag.maxRms                      =   5
prepare_losoto_ion.control.flag.replace                     =   False
prepare_losoto_ion.control.flag.preFlagZeros                =   False
prepare_losoto_ion.control.flag.mode                        =   smooth
prepare_losoto_ion.control.flagextend.operation             =   FLAGEXTEND
prepare_losoto_ion.control.flagextend.soltab                =   sol000/amplitude000
prepare_losoto_ion.control.flagextend.axesToExt             =   [time,freq]
prepare_losoto_ion.control.flagextend.size                  =   [200,80]
prepare_losoto_ion.control.flagextend.percent               =   50
prepare_losoto_ion.control.flagextend.maxCycles             =   2
prepare_losoto_ion.control.plotA2.operation                 =   PLOT
prepare_losoto_ion.control.plotA2.soltab                    =   sol000/amplitude000
prepare_losoto_ion.control.plotA2.axesInPlot                =   [time,freq]
prepare_losoto_ion.control.plotA2.axisInTable               =   ant
prepare_losoto_ion.control.plotA2.plotFlag                  =   True
prepare_losoto_ion.control.plotA2.prefix                    =   {{ inspection_directory }}/ion_ampAFlag_
prepare_losoto_ion.control.merge.operation                  =   REWEIGHT
prepare_losoto_ion.control.merge.mode                       =   copy
prepare_losoto_ion.control.merge.soltab                     =   sol000/phase000
prepare_losoto_ion.control.merge.soltabImport               =   amplitude000
prepare_losoto_ion.control.plotP3.operation                 =   PLOT
prepare_losoto_ion.control.plotP3.soltab                    =   sol000/phase000
prepare_losoto_ion.control.plotP3.axesInPlot                =   [time,freq]
prepare_losoto_ion.control.plotP3.axisInTable               =   ant
prepare_losoto_ion.control.plotP3.plotFlag                  =   True
prepare_losoto_ion.control.plotP3.prefix                    =   {{ inspection_directory }}/ion_ph_
prepare_losoto_ion.control.plotP3.refAnt                    =   {{ refant }}
prepare_losoto_ion.control.plotP3.minmax                    =   [-3.14,3.14]
prepare_losoto_ion.control.plotPd.operation                 =   PLOT
prepare_losoto_ion.control.plotPd.soltab                    =   sol000/phase000
prepare_losoto_ion.control.plotPd.axesInPlot                =   [time,freq]
prepare_losoto_ion.control.plotPd.axisInTable               =   ant
prepare_losoto_ion.control.plotPd.axisDiff                  =   pol
prepare_losoto_ion.control.plotPd.plotFlag                  =   True
prepare_losoto_ion.control.plotPd.prefix                    =   {{ inspection_directory }}/ion_ph_poldif
prepare_losoto_ion.control.plotPd.refAnt                    =   {{ refant }}
prepare_losoto_ion.control.plotPd.minmax                    =   [-3.14,3.14]
prepare_losoto_ion.control.duplicatePbkp.operation          =   DUPLICATE
prepare_losoto_ion.control.duplicatePbkp.soltab             =   sol000/phase000
prepare_losoto_ion.control.duplicatePbkp.soltabOut          =   phaseOrig
prepare_losoto_ion.control.ct.operation                     =   CLOCKTEC
prepare_losoto_ion.control.ct.soltab                        =   sol000/phase000
prepare_losoto_ion.control.ct.clocksoltabOut                =   clock
prepare_losoto_ion.control.ct.tecsoltabOut                  =   tec
prepare_losoto_ion.control.ct.tec3rdsoltabOut               =   tec3rd
prepare_losoto_ion.control.ct.offsetsoltabOut               =   phase_offset
prepare_losoto_ion.control.ct.CombinePol                    =   True
prepare_losoto_ion.control.ct.FlagBadChannels               =   False
prepare_losoto_ion.control.ct.Fit3rdOrder                   =   False
prepare_losoto_ion.control.ct.Circular                      =   False
prepare_losoto_ion.control.ct3.operation                    =   CLOCKTEC
prepare_losoto_ion.control.ct3.soltab                       =   sol000/phase000
prepare_losoto_ion.control.ct3.clocksoltabOut               =   clock
prepare_losoto_ion.control.ct3.tecsoltabOut                 =   tec
prepare_losoto_ion.control.ct3.tec3rdsoltabOut              =   tec3rd
prepare_losoto_ion.control.ct3.offsetsoltabOut              =   phase_offset
prepare_losoto_ion.control.ct3.CombinePol                   =   True
prepare_losoto_ion.control.ct3.FlagBadChannels              =   False
prepare_losoto_ion.control.ct3.Fit3rdOrder                  =   True
prepare_losoto_ion.control.ct3.Circular                     =   False
prepare_losoto_ion.control.plotClock.operation              =   PLOT
prepare_losoto_ion.control.plotClock.soltab                 =   sol000/clock
prepare_losoto_ion.control.plotClock.axesInPlot             =   [time]
prepare_losoto_ion.control.plotClock.axisInTable            =   ant
prepare_losoto_ion.control.plotClock.prefix                 =   {{ inspection_directory }}/clock
prepare_losoto_ion.control.plotClock.plotFlag               =   False
prepare_losoto_ion.control.plotClock.refAnt                 =   {{ refant }}
prepare_losoto_ion.control.plotTEC.operation                =   PLOT
prepare_losoto_ion.control.plotTEC.soltab                   =   sol000/tec
prepare_losoto_ion.control.plotTEC.axesInPlot               =   [time]
prepare_losoto_ion.control.plotTEC.axisInTable              =   ant
prepare_losoto_ion.control.plotTEC.prefix                   =   {{ inspection_directory }}/tec
prepare_losoto_ion.control.plotTEC.plotFlag                 =   False
prepare_losoto_ion.control.plotTEC.refAnt                   =   {{ refant }}
prepare_losoto_ion.control.plotTEC3.operation               =   PLOT
prepare_losoto_ion.control.plotTEC3.soltab                  =   sol000/tec3rd
prepare_losoto_ion.control.plotTEC3.axesInPlot              =   [time]
prepare_losoto_ion.control.plotTEC3.axisInTable             =   ant
prepare_losoto_ion.control.plotTEC3.prefix                  =   {{ inspection_directory }}/tec3rd
prepare_losoto_ion.control.plotTEC3.plotFlag                =   False
prepare_losoto_ion.control.plotTEC3.refAnt                  =   {{ refant }}
prepare_losoto_ion.control.smooth.operation                 =   SMOOTH
prepare_losoto_ion.control.smooth.soltab                    =   sol000/clock
prepare_losoto_ion.control.smooth.axesToSmooth              =   [time]
prepare_losoto_ion.control.smooth.mode                      =   median
prepare_losoto_ion.control.smooth.log                       =   False
prepare_losoto_ion.control.smooth.replace                   =   True
prepare_losoto_ion.control.residuals.operation              =   RESIDUALS
prepare_losoto_ion.control.residuals.soltab                 =   sol000/phase000
prepare_losoto_ion.control.residuals.soltabsToSub           =   [tec,clock]
prepare_losoto_ion.control.residuals3.operation             =   RESIDUALS
prepare_losoto_ion.control.residuals3.soltab                =   sol000/phase000
prepare_losoto_ion.control.residuals3.soltabsToSub          =   [tec,clock,tec3rd] # only for very low-freq dataset
prepare_losoto_ion.control.plotPr.operation                 =   PLOT
prepare_losoto_ion.control.plotPr.soltab                    =   sol000/phase000
prepare_losoto_ion.control.plotPr.axesInPlot                =   [time,freq]
prepare_losoto_ion.control.plotPr.axisInTable               =   ant
prepare_losoto_ion.control.plotPr.axisDiff                  =   pol
prepare_losoto_ion.control.plotPr.plotFlag                  =   True
prepare_losoto_ion.control.plotPr.prefix                    =   {{ inspection_directory }}/ion_ph-res_poldif
prepare_losoto_ion.control.plotPr.refAnt                    =   {{ refant }}
prepare_losoto_ion.control.plotPr.minmax                    =   [-3.14,3.14]
prepare_losoto_ion.control.plotPr3.operation                =   PLOT
prepare_losoto_ion.control.plotPr3.soltab                   =   sol000/phase000
prepare_losoto_ion.control.plotPr3.axesInPlot               =   [time,freq]
prepare_losoto_ion.control.plotPr3.axisInTable              =   ant
prepare_losoto_ion.control.plotPr3.plotFlag                 =   True
prepare_losoto_ion.control.plotPr3.prefix                   =   {{ inspection_directory }}/ion_ph-res_
prepare_losoto_ion.control.plotPr3.refAnt                   =   {{ refant }}
prepare_losoto_ion.control.plotPr3.minmax                   =   [-3.14,3.14]
prepare_losoto_ion.control.flagstation.operation            =   FLAGSTATION
prepare_losoto_ion.control.flagstation.soltab               =   sol000/phase000
prepare_losoto_ion.control.flagstation.maxStddev            =   {{ maxStddev }}
prepare_losoto_ion.control.flagstation.mode                 =   resid
prepare_losoto_ion.control.flagstation.refAnt               =   {{ refant }}
prepare_losoto_ion.control.flagstation.soltabExport         =   clock

# do the processing on the LoSoTo file
process_losoto_ion.control.kind                             =   recipe
process_losoto_ion.control.type                             =   executable_args
process_losoto_ion.control.inplace                          =   True
process_losoto_ion.control.executable                       =   {{ losoto_directory }}/bin/losoto
process_losoto_ion.control.max_per_node                     =   {{ num_proc_per_node }}
process_losoto_ion.control.mapfile_in                       =   h5imp_cal_ion.output.mapfile
process_losoto_ion.control.inputkey                         =   h5in
process_losoto_ion.argument.flags                           =   [-v,h5in,{{ job_directory }}/losoto.parset]

# output the final soltab into an external h5parm
h5exp_cal_ion.control.kind                                  =   recipe
h5exp_cal_ion.control.type                                  =   executable_args
h5exp_cal_ion.control.inplace                               =   True
h5exp_cal_ion.control.executable                            =   {{ losoto_directory }}/bin/H5parm_collector.py
h5exp_cal_ion.control.error_tolerance                       =   {{ error_tolerance }}
h5exp_cal_ion.control.mapfile_in                            =   h5imp_cal_ion.output.mapfile
h5exp_cal_ion.control.inputkey                              =   h5in
h5exp_cal_ion.argument.flags                                =   [-q,-v,h5in]
h5exp_cal_ion.argument.insoltab                             =   {{ tables2export }}
h5exp_cal_ion.argument.outh5parm                            =   {{ cal_solutions }}
h5exp_cal_ion.argument.outsolset                            =   calibrator


################################
## applying the results       ##
################################
# apply the PA solutions
apply_PA.control.type                                       =   dppp
apply_PA.control.error_tolerance                            =   {{ error_tolerance }}
apply_PA.control.inplace                                    =   True
apply_PA.control.max_per_node                               =   {{ num_proc_per_node_limit }}
apply_PA.control.mapfile_in                                 =   ndppp_prep_cal.output.mapfile
apply_PA.control.inputkey                                   =   msfile
apply_PA.argument.msin                                      =   msfile
apply_PA.argument.numthreads                                =   {{ max_dppp_threads }}
apply_PA.argument.msin.datacolumn                           =   DATA
apply_PA.argument.msout.datacolumn                          =   CORRECTED_DATA
apply_PA.argument.msout.storagemanager                      =   "Dysco"
apply_PA.argument.msout.storagemanager.databitrate          =   0
apply_PA.argument.steps                                     =   [applyPA]
apply_PA.argument.applyPA.type                              =   applycal
apply_PA.argument.applyPA.correction                        =   polalign
apply_PA.argument.applyPA.parmdb                            =   {{ cal_solutions }}

# apply the bandpass
apply_bandpass.control.type                                 =   dppp
apply_bandpass.control.error_tolerance                      =   {{ error_tolerance }}
apply_bandpass.control.inplace                              =   True
apply_bandpass.control.max_per_node                         =   {{ num_proc_per_node_limit }}
apply_bandpass.control.mapfile_in                           =   ndppp_prep_cal.output.mapfile
apply_bandpass.control.inputkey                             =   msfile
apply_bandpass.argument.msin                                =   msfile
apply_bandpass.argument.numthreads                          =   {{ max_dppp_threads }}
apply_bandpass.argument.msin.datacolumn                     =   CORRECTED_DATA
apply_bandpass.argument.msout.datacolumn                    =   CORRECTED_DATA
apply_bandpass.argument.msout.storagemanager                =   "Dysco"
apply_bandpass.argument.msout.storagemanager.databitrate    =   0
apply_bandpass.argument.steps                               =   [applybandpass]
apply_bandpass.argument.applybandpass.type                  =   applycal
apply_bandpass.argument.applybandpass.correction            =   bandpass
apply_bandpass.argument.applybandpass.parmdb                =   {{ cal_solutions }}
apply_bandpass.argument.applybandpass.updateweights         =   True

# apply the beam
apply_beam.control.type                                     =   dppp
apply_beam.control.error_tolerance                          =   {{ error_tolerance }}
apply_beam.control.inplace                                  =   True
apply_beam.control.max_per_node                             =   {{ num_proc_per_node_limit }}
apply_beam.control.mapfile_in                               =   ndppp_prep_cal.output.mapfile
apply_beam.control.inputkey                                 =   msfile
apply_beam.argument.msin                                    =   msfile
apply_beam.argument.numthreads                              =   {{ max_dppp_threads }}
apply_beam.argument.msin.datacolumn                         =   CORRECTED_DATA
apply_beam.argument.msout.datacolumn                        =   CORRECTED_DATA
apply_beam.argument.msout.storagemanager                    =   "Dysco"
apply_beam.argument.msout.storagemanager.databitrate        =   0
apply_beam.argument.steps                                   =   [applybeam]
apply_beam.argument.applybeam.type                          =   applybeam
apply_beam.argument.applybeam.invert                        =   True
apply_beam.argument.applybeam.usechannelfreq                =   False
apply_beam.argument.applybeam.beammode                      =   element
apply_beam.argument.applybeam.updateweights                 =   True

# apply the FR solutions
apply_FR.control.type                                       =   dppp
apply_FR.control.error_tolerance                            =   {{ error_tolerance }}
apply_FR.control.inplace                                    =   True
apply_FR.control.max_per_node                               =   {{ num_proc_per_node_limit }}
apply_FR.control.mapfile_in                                 =   ndppp_prep_cal.output.mapfile
apply_FR.control.inputkey                                   =   msfile
apply_FR.argument.msin                                      =   msfile
apply_FR.argument.numthreads                                =   {{ max_dppp_threads }}
apply_FR.argument.msin.datacolumn                           =   CORRECTED_DATA
apply_FR.argument.msout.datacolumn                          =   CORRECTED_DATA
apply_FR.argument.msout.storagemanager                      =   "Dysco"
apply_FR.argument.msout.storagemanager.databitrate          =   0
apply_FR.argument.steps                                     =   [applyFR]
apply_FR.argument.applyFR.type                              =   applycal
apply_FR.argument.applyFR.correction                        =   faraday
apply_FR.argument.applyFR.parmdb                            =   {{ cal_solutions }}

# apply the clock solutions
apply_clock.control.type                                    =   dppp
apply_clock.control.error_tolerance                         =   {{ error_tolerance }}
apply_clock.control.inplace                                 =   True
apply_clock.control.max_per_node                            =   {{ num_proc_per_node_limit }}
apply_clock.control.mapfile_in                              =   ndppp_prep_cal.output.mapfile
apply_clock.control.inputkey                                =   msfile
apply_clock.argument.msin                                   =   msfile
apply_clock.argument.numthreads                             =   {{ max_dppp_threads }}
apply_clock.argument.msin.datacolumn                        =   CORRECTED_DATA
apply_clock.argument.msout.datacolumn                       =   CORRECTED_DATA
apply_clock.argument.msout.storagemanager                   =   "Dysco"
apply_clock.argument.msout.storagemanager.databitrate       =   0
apply_clock.argument.steps                                  =   [applyclock]
apply_clock.argument.applyclock.type                        =   applycal
apply_clock.argument.applyclock.correction                  =   clock
apply_clock.argument.applyclock.parmdb                      =   {{ cal_solutions }}

# apply the TEC solutions
apply_TEC.control.type                                      =   dppp
apply_TEC.control.error_tolerance                           =   {{ error_tolerance }}
apply_TEC.control.inplace                                   =   True
apply_TEC.control.max_per_node                              =   {{ num_proc_per_node_limit }}
apply_TEC.control.mapfile_in                                =   ndppp_prep_cal.output.mapfile
apply_TEC.control.inputkey                                  =   msfile
apply_TEC.argument.msin                                     =   msfile
apply_TEC.argument.numthreads                               =   {{ max_dppp_threads }}
apply_TEC.argument.msin.datacolumn                          =   CORRECTED_DATA
apply_TEC.argument.msout.datacolumn                         =   CORRECTED_DATA
apply_TEC.argument.msout.storagemanager                     =   "Dysco"
apply_TEC.argument.msout.storagemanager.databitrate         =   0
apply_TEC.argument.steps                                    =   [applytec]
apply_TEC.argument.applytec.type                            =   applycal
apply_TEC.argument.applytec.correction                      =   tec
apply_TEC.argument.applytec.parmdb                          =   {{ cal_solutions }}

# # apply the phase offset solutions
# apply_offset.control.type                                   =   dppp
# apply_offset.control.error_tolerance                        =   {{ error_tolerance }}
# apply_offset.control.inplace                                =   True
# apply_offset.control.max_per_node                           =   {{ num_proc_per_node_limit }}
# apply_offset.control.mapfile_in                             =   ndppp_prep_cal.output.mapfile
# apply_offset.control.inputkey                               =   msfile
# apply_offset.argument.msin                                  =   msfile
# apply_offset.argument.numthreads                            =   {{ max_dppp_threads }}
# apply_offset.argument.msin.datacolumn                       =   CORRECTED_DATA
# apply_offset.argument.msout.datacolumn                      =   CORRECTED_DATA
# apply_offset.argument.msout.storagemanager                  =   "Dysco"
# apply_offset.argument.msout.storagemanager.databitrate      =   0
# apply_offset.argument.steps                                 =   [applyoffset]
# apply_offset.argument.applyoffset.type                      =   applycal
# apply_offset.argument.applyoffset.correction                =   phase_offset
# apply_offset.argument.applyoffset.parmdb                    =   {{ cal_solutions }}

###############################
## finalizing the results     ##
################################
# set the pointing direction
h5parm_name.control.type                                    =   pythonplugin
h5parm_name.control.executable                              =   {{ scripts }}/h5parm_pointingname.py
h5parm_name.control.error_tolerance                         =   {{ error_tolerance }}
h5parm_name.argument.flags                                  =   [{{ cal_solutions }}]
h5parm_name.argument.solsetName                             =   calibrator
h5parm_name.argument.pointing                               =   sky_cal.output.SkymodelName.mapfile

# set the pointing direction
make_summary.control.type                                   =   pythonplugin
make_summary.control.executable                             =   {{ scripts }}/make_summary.py
make_summary.control.error_tolerance                        =   {{ error_tolerance }}
make_summary.control.mapfile_in                             =   combine_data_cal_map.output.mapfile
make_summary.control.inputkey                               =   infiles
make_summary.argument.observation_directory                 =   {{ working_directory }}
make_summary.argument.logfile                               =   {{ log_file }}
make_summary.argument.h5parmdb                              =   {{ cal_solutions }}
make_summary.argument.inspection_directory                  =   {{ inspection_directory }}
make_summary.argument.MSfile                                =   infiles

########################################################
##                                                    ##
##                  END PIPELINE                      ##
##                                                    ##
########################################################
