########################################################################## # Pre-Facet Calibrator Calibration Pipeline v3.0 (04/09/2019) # # # # Calibrator part of the basic Pre-Facet calibration pipeline: # # - requires LOFAR software version >= 3.1.0 # # - requires losoto software version >= 2.0.0 # # - expects shared filesystem, that all nodes can reach all files! # # (E.g. a single workstation or compute cluster with shared filesystem # # doesn't work on multiple nodes on CEP3.) # ########################################################################## ########################################## ### parameters you will need to adjust. ## ########################################## ## information about the calibrator data ! cal_input_path = /input_data/calib ## specify the directory where your calibrator data is stored ! cal_input_pattern = *.MS ## regular expression pattern of all your calibrator files ## location of the software ! prefactor_directory = /opt/prefactor/ ## path to your prefactor copy ! losoto_directory = /opt/lofarsoft ## path to your local LoSoTo installation ! aoflagger = /opt/lofarsoft/bin/aoflagger ## path to your aoflagger executable ########################################## ### parameters you may need to adjust ## ########################################## ! refant = 'CS001HBA0' ## name of the station that will be used as a reference for the phase-plots, 'closest' will reference to the spatially closest unflagged antenna ! flag_baselines = [] ## NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] ) ! process_baselines_cal = *& ## performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations. ! filter_baselines = {{ process_baselines_cal }} ## selects only this set of baselines to be processed. Choose [CR]S*& if you want to process only cross-correlations and remove international stations. ! do_smooth = False ## enable or disable baseline-based smoothing ! rfistrategy = HBAdefault.rfis ## strategy to be applied with the statistical flagger (AOFlagger) ! max2interpolate = 30 ## amount of channels in which interpolation should be performed for deriving the bandpass ! ampRange = [0, 0] ## range of median amplitudes accepted per station. Use [0, 0] to use typical ranges of [50, 225] for the current correlator or [0.0004, 0.0018] for the old one. ! skip_international = True ## skip fitting the bandpass for international stations (this avoids flagging them in many cases) ! raw_data = False ## use autoweight, set to True in case you are using raw data ! propagatesolutions = True ## use already derived solutions as initial guess for the upcoming time slot ! flagunconverged = False ## flag solutions for solves that did not converge (if they were also detected to diverge) ! maxStddev = -1.0 ## Maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes # options for solution transfer (for non-trusted calibrator observations) ! solutions2transfer = {{ prefactor_directory }}/solutions/3C48.h5 ## location of h5parm reference solutions to transfer ! antennas2transfer = [FUSPID].* ## regular expression of antennas which solutions should be transferred to the final solution set, if none of the trusted calibrators have been observed (by default: all international stations) # demixing options (only used if demix step is added to the prep_cal_strategy variable) ! demix_sources = [CasA,CygA] ## choose sources to demix (provided as list) ! demix_target = "" ## if given, the target source model (its patch in the SourceDB) is taken into account when solving ! demix_freqstep = 16 ## number of channels to average when demixing. ! demix_timestep = 10 ## number of time slots to average when demixing # definitions for pipeline options -- do not change! ! default_flagging = flagbaseline,flagelev,flagamp ## regular flagging after pre-processing by the observatory pipelines ! raw_flagging = flagedge,aoflag,{{ default_flagging }} ## full flagging (usually only necessary for raw data) ! 1st_order = ct,plotTEC,residuals ## Do not change! Only cal_ion should be edited if needed ! 3rd_order = ct3,plotTEC,plotTEC3,residuals3 ## Do not change! Only cal_ion should be edited if needed ! demix = demix, ## Do not change! Only demix_step should be edited if needed ! full_apply = ,apply_PA,apply_bandpass,apply_clock,apply_beam,apply_FR,apply_TEC ## Do not change! ! none = ## Do not change! # pipeline options ! initial_flagging = {{ default_flagging }} ## choose {{ raw_flagging }} if you process raw data ! demix_step = {{ none }} ## choose {{ demix }} if you want to demix ! cal_ion = {{ 1st_order }},smooth ## choose {{ 3rd_order }} if you want to include 3rd order ionospheric effects (may be useful for LBA < 35 MHz), add smooth if you want to use the median of the clock in time (suggested for HBA+LB) ! tables2export = clock ## comma-separated list of tables to export from the ionospheric calibration step (cal_ion) ! final_apply = {{ none }} ## choose {{ full_apply }} if you want to apply all extracted solutions to the calibrator field ########################################## ### parameters for pipeline performance ## ########################################## ! num_proc_per_node = input.output.max_per_node ## number of processes to use per step per node (usually max_per_node from pipeline.cfg) ! num_proc_per_node_limit = 4 ## number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration) ! max_dppp_threads = 10 ## number of threads per process for NDPPP ! memoryperc = 20 ## maximum of memory used for aoflagger in raw_flagging mode in percent ! min_length = 50 ## minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read. ! overhead = 0.8 ## Only use this fraction of the available memory for deriving the amount of data to be concatenated. ! min_separation = 30 ## minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING) ! max_separation_arcmin = 1.0 ## maximum distance to the phase center for which a skymodel will be still accepted. Change with caution! ! error_tolerance = False ## set this to True if you want the pipeline run to continue if single bands fail ########################################## ### parameters you may want to adjust ## ########################################## ## main directories ! lofar_directory = $LOFARROOT ## base directory of your LOFAR installation ! job_directory = input.output.job_directory ## directory of the prefactor outputs ! working_directory = input.output.working_directory/input.output.job_name ## specify the working_directory (intermediate data products) ! log_file = input.output.log_file ## location of the logfile ! mapfile_dir = input.output.mapfile_dir ## specify mapfile directory ## script and plugin directories ! scripts = {{ prefactor_directory }}/scripts pipeline.pluginpath = {{ prefactor_directory }}/plugins ## skymodel directory ! calibrator_path_skymodel = {{ prefactor_directory }}/skymodels ! A-team_skymodel = {{ calibrator_path_skymodel }}/Ateam_LBA_CC.skymodel ## result directories ! results_directory = {{ job_directory }}/results ## location of the results ! inspection_directory = {{ results_directory }}/inspection ## directory where the inspection plots will be stored ! cal_values_directory = {{ results_directory }}/cal_values ## directory where the final h5parm solution set will be stored ## calibrator solutions ! cal_solutions = {{ cal_values_directory }}/cal_solutions.h5 ## averaging for the calibrator data ! avg_timeresolution = 4. ## average to 4 sec/timeslot ! avg_freqresolution = 48.82kHz ## average to 48.82 kHz/ch (= 4 ch/SB) ! bandpass_freqresolution = 195.3125kHz ## resolution of the bandpass table is 195.3125kHz (= 1 ch/SB) ######################################################## ## ## ## BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE! ## ## ## ######################################################## # which steps to run pipeline.steps = [prep, PA, FR, bandpass, ion, finalize] # pipeline substeps pipeline.steps.prep = [createmap_cal, combine_data_map, check_Ateam_separation, mk_cal_values_dir, createmap_prepcal, createmap_instcal, create_ateam_model_map, make_sourcedb_ateam, expand_sourcedb_ateam, ndppp_prep_cal, combine_data_cal_map, ms_concat, ms_concat_map, expand_memory_map, aoflag, sky_cal, make_sourcedb, expand_sourcedb, expand_skymodel, calib_cal_parmmap, h5imp_cal_map, smooth_data, predict_cal] pipeline.steps.PA = [calib_cal, h5imp_cal_PA, prepare_losoto_PA, process_losoto_PA, h5exp_cal_PA, apply_PA, apply_beam] pipeline.steps.FR = [smooth_corrected, calib_cal, h5imp_cal_FR, prepare_losoto_FR, process_losoto_FR, h5exp_cal_FR, apply_FR] pipeline.steps.bandpass = [smooth_corrected, calib_cal2, h5imp_cal_bandpass, prepare_losoto_bandpass, prepare_losoto_bandpasstrans, process_losoto_bandpass, h5exp_cal_bandpass, transfer_solutions, apply_PA, apply_bandpass, apply_beam, apply_FR] pipeline.steps.ion = [smooth_corrected, calib_cal2, h5imp_cal_ion, prepare_losoto_ion, process_losoto_ion, h5exp_cal_ion] pipeline.steps.finalize = [h5parm_name {{ final_apply }}, make_summary] ############################### ## Mapping calibrator files ## ############################### # generate a mapfile of all the calibrator data createmap_cal.control.kind = plugin createmap_cal.control.type = createMapfile createmap_cal.control.method = mapfile_from_folder createmap_cal.control.mapfile_dir = {{ mapfile_dir }} createmap_cal.control.filename = createmap_cal.mapfile createmap_cal.control.folder = {{ cal_input_path }} createmap_cal.control.pattern = {{ cal_input_pattern }} # combine all entries into one mapfile, for the sortmap script combine_data_map.control.kind = plugin combine_data_map.control.type = createMapfile combine_data_map.control.method = mapfile_all_to_one combine_data_map.control.mapfile_dir = {{ mapfile_dir }} combine_data_map.control.filename = combine_data_map.mapfile combine_data_map.control.mapfile_in = createmap_cal.output.mapfile # warn for potential nearby A-Team sources check_Ateam_separation.control.type = pythonplugin check_Ateam_separation.control.executable = {{ scripts }}/check_Ateam_separation.py check_Ateam_separation.control.mapfile_in = combine_data_map.output.mapfile check_Ateam_separation.control.inputkey = MSfile check_Ateam_separation.argument.min_separation = {{ min_separation }} check_Ateam_separation.argument.outputimage = {{ inspection_directory }}/A-Team_elevation_calibrator.png check_Ateam_separation.argument.flags = [MSfile] ############################# ## Prepare for demixing ## ############################# # generate a mapfile of the calibrator createmap_prepcal.control.kind = plugin createmap_prepcal.control.type = makeResultsMapfile createmap_prepcal.control.mapfile_dir = {{ mapfile_dir }} createmap_prepcal.control.filename = createmap_prepcal.mapfile createmap_prepcal.control.mapfile_in = createmap_cal.output.mapfile createmap_prepcal.control.target_dir = {{ job_directory }} createmap_prepcal.control.make_target_dir = False createmap_prepcal.control.new_suffix = .ndppp_prep_cal # generate a mapfile for the instrument table of the calibrator createmap_instcal.control.kind = plugin createmap_instcal.control.type = changeMapfile createmap_instcal.control.mapfile_in = createmap_prepcal.output.mapfile createmap_instcal.control.join_files = instrument createmap_instcal.control.newname = createmap_instcal.mapfile # create a mapfile with the A-Team skymodel, length = 1 create_ateam_model_map.control.kind = plugin create_ateam_model_map.control.type = addListMapfile create_ateam_model_map.control.hosts = ['localhost'] create_ateam_model_map.control.files = [ {{ A-team_skymodel }} ] create_ateam_model_map.control.mapfile_dir = {{ mapfile_dir }} create_ateam_model_map.control.filename = ateam_model_name.mapfile # make sourcedbs from the A-Team skymodel, length = 1 make_sourcedb_ateam.control.kind = recipe make_sourcedb_ateam.control.type = executable_args make_sourcedb_ateam.control.executable = {{ lofar_directory }}/bin/makesourcedb make_sourcedb_ateam.control.error_tolerance = {{ error_tolerance }} make_sourcedb_ateam.control.args_format = lofar make_sourcedb_ateam.control.outputkey = out make_sourcedb_ateam.control.mapfile_in = create_ateam_model_map.output.mapfile make_sourcedb_ateam.control.inputkey = in make_sourcedb_ateam.argument.format = < make_sourcedb_ateam.argument.outtype = blob # expand the sourcedb mapfile so that there is one entry for every file, length = nfiles expand_sourcedb_ateam.control.kind = plugin expand_sourcedb_ateam.control.type = expandMapfile expand_sourcedb_ateam.control.mapfile_in = make_sourcedb_ateam.output.mapfile expand_sourcedb_ateam.control.mapfile_to_match = createmap_cal.output.mapfile expand_sourcedb_ateam.control.mapfile_dir = {{ mapfile_dir }} expand_sourcedb_ateam.control.filename = expand_sourcedb_ateam.datamap ############################### ## Prepare calibrator ## ############################### # create the cal_values_directory if needed mk_cal_values_dir.control.kind = plugin mk_cal_values_dir.control.type = makeDirectory mk_cal_values_dir.control.directory = {{ cal_values_directory }} # run NDPPP on the calibrator data ndppp_prep_cal.control.type = dppp ndppp_prep_cal.control.max_per_node = {{ num_proc_per_node_limit }} ndppp_prep_cal.control.error_tolerance = {{ error_tolerance }} ndppp_prep_cal.argument.numthreads = {{ max_dppp_threads }} ndppp_prep_cal.argument.msin = createmap_cal.output.mapfile # The input data. ndppp_prep_cal.argument.msin.datacolumn = DATA ndppp_prep_cal.argument.msin.baseline = {{ filter_baselines }} ndppp_prep_cal.argument.msin.autoweight = {{ raw_data }} ndppp_prep_cal.argument.msout.datacolumn = DATA ndppp_prep_cal.argument.msout.writefullresflag = False ndppp_prep_cal.argument.msout.overwrite = True ndppp_prep_cal.argument.msout.storagemanager = "Dysco" ndppp_prep_cal.argument.msout.storagemanager.databitrate = 0 ndppp_prep_cal.argument.steps = [{{ initial_flagging }},{{ demix_step }}avg] ndppp_prep_cal.argument.flagedge.type = preflagger ndppp_prep_cal.argument.flagedge.chan = [0..nchan/32-1,31*nchan/32..nchan-1] # we are running on a single subband ndppp_prep_cal.argument.aoflag.type = aoflagger ndppp_prep_cal.argument.aoflag.memoryperc = {{ memoryperc }} ndppp_prep_cal.argument.aoflag.keepstatistics = false ndppp_prep_cal.argument.flagbaseline.type = preflagger ndppp_prep_cal.argument.flagbaseline.baseline = {{ flag_baselines }} ndppp_prep_cal.argument.flagelev.type = preflagger ndppp_prep_cal.argument.flagelev.elevation = 0deg..20deg ndppp_prep_cal.argument.flagamp.type = preflagger ndppp_prep_cal.argument.flagamp.amplmin = 1e-30 ndppp_prep_cal.argument.avg.type = average ndppp_prep_cal.argument.avg.timeresolution = {{ avg_timeresolution }} ndppp_prep_cal.argument.avg.freqresolution = {{ avg_freqresolution }} ndppp_prep_cal.argument.flagamp.type = preflagger ndppp_prep_cal.argument.flagamp.amplmin = 1e-30 ndppp_prep_cal.argument.demix.type = demixer ndppp_prep_cal.argument.demix.baseline = {{ process_baselines_cal }} ndppp_prep_cal.argument.demix.demixfreqstep = {{ demix_freqstep }} ndppp_prep_cal.argument.demix.demixtimestep = {{ demix_timestep }} ndppp_prep_cal.argument.demix.ignoretarget = False ndppp_prep_cal.argument.demix.targetsource = {{ demix_target }} ndppp_prep_cal.argument.demix.subtractsources = {{ demix_sources }} ndppp_prep_cal.argument.demix.ntimechunk = {{ max_dppp_threads }} ndppp_prep_cal.argument.demix.skymodel = expand_sourcedb_ateam.output.mapfile ndppp_prep_cal.argument.demix.freqstep = 1 ndppp_prep_cal.argument.demix.timestep = 1 ndppp_prep_cal.argument.demix.instrumentmodel = createmap_instcal.output.mapfile # combine all entries into one mapfile (just for the find_skymodel_cal_auto script) combine_data_cal_map.control.kind = plugin combine_data_cal_map.control.type = createMapfile combine_data_cal_map.control.method = mapfile_all_to_one combine_data_cal_map.control.mapfile_dir = {{ mapfile_dir }} combine_data_cal_map.control.filename = combine_data_cal_map.mapfile combine_data_cal_map.control.mapfile_in = ndppp_prep_cal.output.mapfile # virtually concatenate calibrator subbands ms_concat.control.type = pythonplugin ms_concat.control.executable = {{ scripts }}/concat_MS.py ms_concat.control.error_tolerance = {{ error_tolerance }} ms_concat.argument.filename = concatmapfile.mapfile ms_concat.argument.mapfile_dir = {{ mapfile_dir }} ms_concat.argument.min_length = {{ min_length }} ms_concat.argument.overhead = {{ overhead }} ms_concat.argument.flags = [combine_data_cal_map.output.mapfile,outputkey] # convert the output of ms_concat into usable mapfiles ms_concat_map.control.kind = plugin ms_concat_map.control.type = mapfilenamesFromMapfiles ms_concat_map.control.mapfile_concatmap = ms_concat.output.concatmapfile.mapfile # convert the output of ms_concat_target into usable mapfiles expand_memory_map.control.kind = plugin expand_memory_map.control.type = expandMapfile expand_memory_map.control.mapfile_in = ms_concat.output.memory.mapfile expand_memory_map.control.mapfile_to_match = ms_concat_map.output.concatmap expand_memory_map.control.mapfile_dir = {{ mapfile_dir }} expand_memory_map.control.filename = expand_memory_map.mapfile # run aoflagger on the concatenated data aoflag.control.kind = recipe aoflag.control.type = executable_args aoflag.control.inplace = True aoflag.control.executable = {{ aoflagger }} aoflag.control.max_per_node = 1 aoflag.control.error_tolerance = {{ error_tolerance }} aoflag.control.mapfiles_in = [ms_concat_map.output.concatmap,expand_memory_map.output.mapfile] aoflag.control.inputkeys = [msin,memory] aoflag.control.args_format = wsclean aoflag.argument.strategy = {{ prefactor_directory }}/rfistrategies/{{ rfistrategy }} aoflag.argument.flags = [-v,memory,-combine-spws,msin] # find automatically the calibrator sky model sky_cal.control.type = pythonplugin sky_cal.control.executable = {{ scripts }}/find_skymodel_cal.py sky_cal.control.error_tolerance = {{ error_tolerance }} sky_cal.argument.flags = [combine_data_cal_map.output.mapfile] sky_cal.argument.DirSkymodelCal = {{ calibrator_path_skymodel }} sky_cal.argument.max_separation_arcmin = {{ max_separation_arcmin }} # make the sourcedb make_sourcedb.control.kind = recipe make_sourcedb.control.type = executable_args make_sourcedb.control.executable = {{ lofar_directory }}/bin/makesourcedb make_sourcedb.control.error_tolerance = {{ error_tolerance }} make_sourcedb.control.args_format = lofar make_sourcedb.control.outputkey = out make_sourcedb.control.mapfile_in = sky_cal.output.SkymodelCal.mapfile make_sourcedb.control.inputkey = in make_sourcedb.argument.format = < make_sourcedb.argument.outtype = blob # expand the sourcedb mapfile so that there is one entry for every file, length = nfiles expand_sourcedb.control.kind = plugin expand_sourcedb.control.type = expandMapfile expand_sourcedb.control.mapfile_in = make_sourcedb.output.mapfile expand_sourcedb.control.mapfile_to_match = ndppp_prep_cal.output.mapfile expand_sourcedb.control.mapfile_dir = {{ mapfile_dir }} expand_sourcedb.control.filename = expand_sourcedb.mapfile # expand the sourcedb mapfile so that there is one entry for every file, length = nfiles expand_skymodel.control.kind = plugin expand_skymodel.control.type = expandMapfile expand_skymodel.control.mapfile_in = sky_cal.output.SkymodelName.mapfile expand_skymodel.control.mapfile_to_match = ndppp_prep_cal.output.mapfile expand_skymodel.control.mapfile_dir = {{ mapfile_dir }} expand_skymodel.control.filename = expand_skymodel.mapfile # generate mapfile with the h5parm names to be used in the calib_cal steps calib_cal_parmmap.control.kind = plugin calib_cal_parmmap.control.type = createMapfile calib_cal_parmmap.control.method = add_suffix_to_file calib_cal_parmmap.control.mapfile_in = ndppp_prep_cal.output.mapfile calib_cal_parmmap.control.add_suffix_to_file = /instrument.h5 calib_cal_parmmap.control.mapfile_dir = {{ mapfile_dir }} calib_cal_parmmap.control.filename = calib_cal_h5parms.mapfile # generate a mapfile with all files in a single entry h5imp_cal_map.control.kind = plugin h5imp_cal_map.control.type = compressMapfile h5imp_cal_map.control.mapfile_in = calib_cal_parmmap.output.mapfile h5imp_cal_map.control.mapfile_dir = {{ mapfile_dir }} h5imp_cal_map.control.filename = h5imp_cal_map.mapfile # baseline-dependent smoothing smooth_data.control.type = executable_args smooth_data.control.inplace = True smooth_data.control.max_per_node = {{ num_proc_per_node }} smooth_data.control.error_tolerance = {{ error_tolerance }} smooth_data.control.executable = {{ scripts }}/BLsmooth.py smooth_data.control.mapfile_in = ndppp_prep_cal.output.mapfile smooth_data.control.inputkey = msin smooth_data.argument.flags = [-S,{{ do_smooth }},-r,-i,DATA,-o,SMOOTHED_DATA,msin] # baseline-dependent smoothing smooth_corrected.control.type = executable_args smooth_corrected.control.inplace = True smooth_corrected.control.max_per_node = {{ num_proc_per_node }} smooth_corrected.control.error_tolerance = {{ error_tolerance }} smooth_corrected.control.executable = {{ scripts }}/BLsmooth.py smooth_corrected.control.mapfile_in = ndppp_prep_cal.output.mapfile smooth_corrected.control.inputkey = msin smooth_corrected.argument.flags = [-S,{{ do_smooth }},-r,-i,CORRECTED_DATA,-o,SMOOTHED_DATA,msin] # predict to save time predict_cal.control.type = dppp predict_cal.control.inplace = True predict_cal.control.max_per_node = {{ num_proc_per_node_limit }} predict_cal.control.error_tolerance = {{ error_tolerance }} predict_cal.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,expand_sourcedb.output.mapfile,expand_skymodel.output.mapfile] predict_cal.control.inputkeys = [msfile,sourcedb,skymodel] predict_cal.argument.msin = msfile predict_cal.argument.numthreads = {{ max_dppp_threads }} predict_cal.argument.msin.datacolumn = SMOOTHED_DATA predict_cal.argument.msout.datacolumn = MODEL_DATA predict_cal.argument.msout.storagemanager = "Dysco" predict_cal.argument.msout.storagemanager.databitrate = 0 predict_cal.argument.steps = [predict] predict_cal.argument.predict.type = predict predict_cal.argument.predict.sourcedb = sourcedb predict_cal.argument.predict.sources = skymodel predict_cal.argument.predict.usebeammodel = False # TODO: put to true for large sky predict_cal.argument.predict.usechannelfreq = False predict_cal.argument.predict.beammode = array_factor # now run NDPPP on the averaged calibrator data (rotation+diagonal) calib_cal.control.type = dppp calib_cal.control.inplace = True calib_cal.control.max_per_node = {{ num_proc_per_node_limit }} calib_cal.control.error_tolerance = {{ error_tolerance }} calib_cal.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile] calib_cal.control.inputkeys = [msfile,h5parm] calib_cal.argument.msin = msfile calib_cal.argument.numthreads = {{ max_dppp_threads }} calib_cal.argument.msin.datacolumn = SMOOTHED_DATA calib_cal.argument.steps = [solve] calib_cal.argument.solve.type = ddecal calib_cal.argument.solve.mode = rotation+diagonal calib_cal.argument.solve.h5parm = h5parm calib_cal.argument.solve.usemodelcolumn = True calib_cal.argument.solve.uvlambdamin = 300 calib_cal.argument.solve.maxiter = 50 calib_cal.argument.solve.nchan = 1 calib_cal.argument.solve.solint = 1 calib_cal.argument.solve.propagatesolutions = {{ propagatesolutions }} calib_cal.argument.solve.propagateconvergedonly = True calib_cal.argument.solve.flagunconverged = {{ flagunconverged }} calib_cal.argument.solve.flagdivergedonly = True calib_cal.argument.solve.tolerance = 1e-3 # now run NDPPP on the averaged calibrator data (diagonal only) calib_cal2.control.type = dppp calib_cal2.control.inplace = True calib_cal2.control.max_per_node = {{ num_proc_per_node_limit }} calib_cal2.control.error_tolerance = {{ error_tolerance }} calib_cal2.control.mapfiles_in = [ndppp_prep_cal.output.mapfile,calib_cal_parmmap.output.mapfile] calib_cal2.control.inputkeys = [msfile,h5parm] calib_cal2.argument.msin = msfile calib_cal2.argument.numthreads = {{ max_dppp_threads }} calib_cal2.argument.msin.datacolumn = SMOOTHED_DATA calib_cal2.argument.steps = [solve] calib_cal2.argument.solve.type = ddecal calib_cal2.argument.solve.mode = diagonal calib_cal2.argument.solve.h5parm = h5parm calib_cal2.argument.solve.usemodelcolumn = True calib_cal2.argument.solve.uvlambdamin = 300 calib_cal2.argument.solve.maxiter = 50 calib_cal2.argument.solve.nchan = 1 calib_cal2.argument.solve.solint = 1 calib_cal2.argument.solve.propagatesolutions = {{ propagatesolutions }} calib_cal2.argument.solve.propagateconvergedonly = True calib_cal2.argument.solve.flagunconverged = {{ flagunconverged }} calib_cal2.argument.solve.flagdivergedonly = True calib_cal2.argument.solve.tolerance = 1e-3 ########################### ## calibrate Pol. Align ## ########################### # collect all instrument tables into one h5parm h5imp_cal_PA.control.kind = recipe h5imp_cal_PA.control.type = executable_args h5imp_cal_PA.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5imp_cal_PA.control.error_tolerance = {{ error_tolerance }} h5imp_cal_PA.control.mapfile_in = h5imp_cal_map.output.mapfile h5imp_cal_PA.control.inputkey = h5parm h5imp_cal_PA.control.outputkey = outh5parm h5imp_cal_PA.argument.flags = [-q,-v,-c,h5parm] h5imp_cal_PA.argument.outh5parm = outh5parm # create losoto v2 parset file prepare_losoto_PA.control.kind = plugin prepare_losoto_PA.control.type = makeLosotoParset prepare_losoto_PA.control.steps = [plotP3,plotPd,plotRot3,plotA3,bkp,align,plotAlign,residual,plotPr,plotPr2] prepare_losoto_PA.control.filename = {{ job_directory }}/losoto.parset prepare_losoto_PA.control.global.ncpu = {{ num_proc_per_node }} prepare_losoto_PA.control.plotP3.operation = PLOT prepare_losoto_PA.control.plotP3.soltab = sol000/phase000 prepare_losoto_PA.control.plotP3.axesInPlot = [time,freq] prepare_losoto_PA.control.plotP3.axisInTable = ant prepare_losoto_PA.control.plotP3.plotFlag = True prepare_losoto_PA.control.plotP3.prefix = {{ inspection_directory }}/polalign_ph_ prepare_losoto_PA.control.plotP3.refAnt = {{ refant }} prepare_losoto_PA.control.plotP3.minmax = [-3.14,3.14] prepare_losoto_PA.control.plotPd.operation = PLOT prepare_losoto_PA.control.plotPd.soltab = sol000/phase000 prepare_losoto_PA.control.plotPd.axesInPlot = [time,freq] prepare_losoto_PA.control.plotPd.axisInTable = ant prepare_losoto_PA.control.plotPd.axisDiff = pol prepare_losoto_PA.control.plotPd.plotFlag = True prepare_losoto_PA.control.plotPd.prefix = {{ inspection_directory }}/polalign_ph_poldif prepare_losoto_PA.control.plotPd.refAnt = {{ refant }} prepare_losoto_PA.control.plotPd.minmax = [-3.14,3.14] prepare_losoto_PA.control.plotRot3.operation = PLOT prepare_losoto_PA.control.plotRot3.soltab = sol000/rotation000 prepare_losoto_PA.control.plotRot3.axesInPlot = [time,freq] prepare_losoto_PA.control.plotRot3.axisInTable = ant prepare_losoto_PA.control.plotRot3.plotFlag = True prepare_losoto_PA.control.plotRot3.prefix = {{ inspection_directory }}/polalign_rotangle prepare_losoto_PA.control.plotRot3.refAnt = {{ refant }} prepare_losoto_PA.control.plotA3.operation = PLOT prepare_losoto_PA.control.plotA3.soltab = sol000/amplitude000 prepare_losoto_PA.control.plotA3.axesInPlot = [time,freq] prepare_losoto_PA.control.plotA3.axisInTable = ant prepare_losoto_PA.control.plotA3.plotFlag = True prepare_losoto_PA.control.plotA3.prefix = {{ inspection_directory }}/polalign_amp_ prepare_losoto_PA.control.bkp.operation = DUPLICATE prepare_losoto_PA.control.bkp.soltab = sol000/phase000 prepare_losoto_PA.control.bkp.soltabOut = phaseOrig prepare_losoto_PA.control.align.soltab = sol000/phase000 prepare_losoto_PA.control.align.operation = POLALIGN prepare_losoto_PA.control.align.soltabOut = polalign prepare_losoto_PA.control.align.average = True prepare_losoto_PA.control.align.replace = True prepare_losoto_PA.control.align.fitOffset = False prepare_losoto_PA.control.align.refAnt = {{ refant }} prepare_losoto_PA.control.plotAlign.operation = PLOT prepare_losoto_PA.control.plotAlign.soltab = sol000/polalign prepare_losoto_PA.control.plotAlign.axesInPlot = [time,freq] prepare_losoto_PA.control.plotAlign.axisInTable = ant prepare_losoto_PA.control.plotAlign.axisDiff = pol prepare_losoto_PA.control.plotAlign.plotFlag = True prepare_losoto_PA.control.plotAlign.prefix = {{ inspection_directory }}/polalign prepare_losoto_PA.control.plotAlign.refAnt = {{ refant }} prepare_losoto_PA.control.plotAlign.minmax = [-3.14,3.14] prepare_losoto_PA.control.residual.operation = RESIDUALS prepare_losoto_PA.control.residual.soltab = sol000/phase000 prepare_losoto_PA.control.residual.soltabsToSub = polalign prepare_losoto_PA.control.plotPr.operation = PLOT prepare_losoto_PA.control.plotPr.soltab = sol000/phase000 prepare_losoto_PA.control.plotPr.axesInPlot = [time,freq] prepare_losoto_PA.control.plotPr.axisInTable = ant prepare_losoto_PA.control.plotPr.axisDiff = pol prepare_losoto_PA.control.plotPr.plotFlag = True prepare_losoto_PA.control.plotPr.prefix = {{ inspection_directory }}/polalign_ph-res_poldif prepare_losoto_PA.control.plotPr.refAnt = {{ refant }} prepare_losoto_PA.control.plotPr.minmax = [-3.14,3.14] prepare_losoto_PA.control.plotPr2.operation = PLOT prepare_losoto_PA.control.plotPr2.soltab = sol000/phase000 prepare_losoto_PA.control.plotPr2.axesInPlot = [time,freq] prepare_losoto_PA.control.plotPr2.axisInTable = ant prepare_losoto_PA.control.plotPr2.axisInCol = pol prepare_losoto_PA.control.plotPr2.plotFlag = True prepare_losoto_PA.control.plotPr2.prefix = {{ inspection_directory }}/polalign_ph-res_ prepare_losoto_PA.control.plotPr2.refAnt = {{ refant }} prepare_losoto_PA.control.plotPr2.minmax = [-3.14,3.14] # do the processing on the LoSoTo file process_losoto_PA.control.kind = recipe process_losoto_PA.control.type = executable_args process_losoto_PA.control.inplace = True process_losoto_PA.control.executable = {{ losoto_directory }}/bin/losoto process_losoto_PA.control.max_per_node = {{ num_proc_per_node }} process_losoto_PA.control.mapfile_in = h5imp_cal_PA.output.mapfile process_losoto_PA.control.inputkey = h5in process_losoto_PA.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] # output the final soltab into an external h5parm h5exp_cal_PA.control.kind = recipe h5exp_cal_PA.control.type = executable_args h5exp_cal_PA.control.inplace = True h5exp_cal_PA.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5exp_cal_PA.control.error_tolerance = {{ error_tolerance }} h5exp_cal_PA.control.mapfile_in = h5imp_cal_PA.output.mapfile h5exp_cal_PA.control.inputkey = h5in h5exp_cal_PA.argument.flags = [-q,-v,h5in] h5exp_cal_PA.argument.insoltab = polalign h5exp_cal_PA.argument.outh5parm = {{ cal_solutions }} h5exp_cal_PA.argument.outsolset = calibrator ################################ ## calibrate Farady Rotation ## ################################ # collect all instrument tables into one h5parm h5imp_cal_FR.control.kind = recipe h5imp_cal_FR.control.type = executable_args h5imp_cal_FR.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5imp_cal_FR.control.error_tolerance = {{ error_tolerance }} h5imp_cal_FR.control.mapfile_in = h5imp_cal_map.output.mapfile h5imp_cal_FR.control.inputkey = h5parm h5imp_cal_FR.control.outputkey = outh5parm h5imp_cal_FR.argument.flags = [-q,-v,-c,h5parm] h5imp_cal_FR.argument.outh5parm = outh5parm # create losoto v2 parset file prepare_losoto_FR.control.kind = plugin prepare_losoto_FR.control.type = makeLosotoParset prepare_losoto_FR.control.steps = [plotP3,plotPd,plotRot3,plotA3,faraday,plotFR,residual,plotPr,plotPr2] prepare_losoto_FR.control.filename = {{ job_directory }}/losoto.parset prepare_losoto_FR.control.global.ncpu = {{ num_proc_per_node }} prepare_losoto_FR.control.plotP3.operation = PLOT prepare_losoto_FR.control.plotP3.soltab = sol000/phase000 prepare_losoto_FR.control.plotP3.axesInPlot = [time,freq] prepare_losoto_FR.control.plotP3.axisInTable = ant prepare_losoto_FR.control.plotP3.plotFlag = True prepare_losoto_FR.control.plotP3.prefix = {{ inspection_directory }}/fr_ph_ prepare_losoto_FR.control.plotP3.refAnt = {{ refant }} prepare_losoto_FR.control.plotP3.minmax = [-3.14,3.14] prepare_losoto_FR.control.plotPd.operation = PLOT prepare_losoto_FR.control.plotPd.soltab = sol000/phase000 prepare_losoto_FR.control.plotPd.axesInPlot = [time,freq] prepare_losoto_FR.control.plotPd.axisInTable = ant prepare_losoto_FR.control.plotPd.axisDiff = pol prepare_losoto_FR.control.plotPd.plotFlag = True prepare_losoto_FR.control.plotPd.prefix = {{ inspection_directory }}/fr_ph_poldif prepare_losoto_FR.control.plotPd.refAnt = {{ refant }} prepare_losoto_FR.control.plotPd.minmax = [-3.14,3.14] prepare_losoto_FR.control.plotRot3.operation = PLOT prepare_losoto_FR.control.plotRot3.soltab = sol000/rotation000 prepare_losoto_FR.control.plotRot3.axesInPlot = [time,freq] prepare_losoto_FR.control.plotRot3.axisInTable = ant prepare_losoto_FR.control.plotRot3.plotFlag = True prepare_losoto_FR.control.plotRot3.prefix = {{ inspection_directory }}/fr_rotangle prepare_losoto_FR.control.plotRot3.refAnt = {{ refant }} prepare_losoto_FR.control.plotA3.operation = PLOT prepare_losoto_FR.control.plotA3.soltab = sol000/amplitude000 prepare_losoto_FR.control.plotA3.axesInPlot = [time,freq] prepare_losoto_FR.control.plotA3.axisInTable = ant prepare_losoto_FR.control.plotA3.plotFlag = True prepare_losoto_FR.control.plotA3.prefix = {{ inspection_directory }}/fr_amp_ prepare_losoto_FR.control.faraday.operation = FARADAY prepare_losoto_FR.control.faraday.soltab = sol000/rotation000 prepare_losoto_FR.control.faraday.soltabOut = faraday prepare_losoto_FR.control.faraday.refAnt = {{ refant }} prepare_losoto_FR.control.faraday.maxResidual = 1. prepare_losoto_FR.control.plotFR.operation = PLOT prepare_losoto_FR.control.plotFR.soltab = sol000/faraday prepare_losoto_FR.control.plotFR.axesInPlot = [time] prepare_losoto_FR.control.plotFR.axisInTable = ant prepare_losoto_FR.control.plotFR.prefix = {{ inspection_directory }}/fr prepare_losoto_FR.control.residual.operation = RESIDUALS prepare_losoto_FR.control.residual.soltab = sol000/phase000 prepare_losoto_FR.control.residual.soltabsToSub = faraday prepare_losoto_FR.control.plotPr.operation = PLOT prepare_losoto_FR.control.plotPr.soltab = sol000/phase000 prepare_losoto_FR.control.plotPr.axesInPlot = [time,freq] prepare_losoto_FR.control.plotPr.axisInTable = ant prepare_losoto_FR.control.plotPr.axisDiff = pol prepare_losoto_FR.control.plotPr.plotFlag = True prepare_losoto_FR.control.plotPr.prefix = {{ inspection_directory }}/fr_ph-res_poldif prepare_losoto_FR.control.plotPr.refAnt = {{ refant }} prepare_losoto_FR.control.plotPr.minmax = [-3.14,3.14] prepare_losoto_FR.control.plotPr2.operation = PLOT prepare_losoto_FR.control.plotPr2.soltab = sol000/phase000 prepare_losoto_FR.control.plotPr2.axesInPlot = [time,freq] prepare_losoto_FR.control.plotPr2.axisInTable = ant prepare_losoto_FR.control.plotPr2.axisInCol = pol prepare_losoto_FR.control.plotPr2.plotFlag = True prepare_losoto_FR.control.plotPr2.prefix = {{ inspection_directory }}/fr_ph-res_ prepare_losoto_FR.control.plotPr2.refAnt = {{ refant }} prepare_losoto_FR.control.plotPr2.minmax = [-3.14,3.14] # do the processing on the LoSoTo file process_losoto_FR.control.kind = recipe process_losoto_FR.control.type = executable_args process_losoto_FR.control.inplace = True process_losoto_FR.control.executable = {{ losoto_directory }}/bin/losoto process_losoto_FR.control.max_per_node = {{ num_proc_per_node }} process_losoto_FR.control.mapfile_in = h5imp_cal_FR.output.mapfile process_losoto_FR.control.inputkey = h5in process_losoto_FR.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] # output the final soltab into an external h5parm h5exp_cal_FR.control.kind = recipe h5exp_cal_FR.control.type = executable_args h5exp_cal_FR.control.inplace = True h5exp_cal_FR.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5exp_cal_FR.control.error_tolerance = {{ error_tolerance }} h5exp_cal_FR.control.mapfile_in = h5imp_cal_FR.output.mapfile h5exp_cal_FR.control.inputkey = h5in h5exp_cal_FR.argument.flags = [-q,-v,h5in] h5exp_cal_FR.argument.insoltab = faraday h5exp_cal_FR.argument.outh5parm = {{ cal_solutions }} h5exp_cal_FR.argument.outsolset = calibrator ################################ ## calibrate bandpass ## ################################ # collect all instrument tables into one h5parm h5imp_cal_bandpass.control.kind = recipe h5imp_cal_bandpass.control.type = executable_args h5imp_cal_bandpass.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5imp_cal_bandpass.control.error_tolerance = {{ error_tolerance }} h5imp_cal_bandpass.control.mapfile_in = h5imp_cal_map.output.mapfile h5imp_cal_bandpass.control.inputkey = h5parm h5imp_cal_bandpass.control.outputkey = outh5parm h5imp_cal_bandpass.argument.flags = [-q,-v,-c,h5parm] h5imp_cal_bandpass.argument.outh5parm = outh5parm # create losoto v2 parset file prepare_losoto_bandpass.control.kind = plugin prepare_losoto_bandpass.control.type = makeLosotoParset prepare_losoto_bandpass.control.steps = [duplicateAbkp,plotA1,flag,flagbp,flagextend,plotA2,merge,smooth,bandpass,interp,smoothb,plotB1,plotB2,plotB3] prepare_losoto_bandpass.control.filename = {{ job_directory }}/losoto.parset prepare_losoto_bandpass.control.global.ncpu = {{ num_proc_per_node }} prepare_losoto_bandpass.control.duplicateAbkp.operation = DUPLICATE prepare_losoto_bandpass.control.duplicateAbkp.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.duplicateAbkp.soltabOut = amplitudeOrig000 prepare_losoto_bandpass.control.plotA1.operation = PLOT prepare_losoto_bandpass.control.plotA1.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.plotA1.axesInPlot = [time,freq] prepare_losoto_bandpass.control.plotA1.axisInTable = ant prepare_losoto_bandpass.control.plotA1.plotFlag = True prepare_losoto_bandpass.control.plotA1.prefix = {{ inspection_directory }}/ampBFlag_ prepare_losoto_bandpass.control.flag.operation = FLAG prepare_losoto_bandpass.control.flag.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.flag.axesToFlag = [time,freq] prepare_losoto_bandpass.control.flag.order = [100,40] prepare_losoto_bandpass.control.flag.maxCycles = 1 prepare_losoto_bandpass.control.flag.maxRms = 5 prepare_losoto_bandpass.control.flag.replace = False prepare_losoto_bandpass.control.flag.preFlagZeros = False prepare_losoto_bandpass.control.flag.mode = smooth prepare_losoto_bandpass.control.flagbp.operation = FLAGSTATION prepare_losoto_bandpass.control.flagbp.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.flagbp.mode = bandpass prepare_losoto_bandpass.control.flagbp.ampRange = {{ ampRange }} prepare_losoto_bandpass.control.flagbp.skipInternational = {{ skip_international }} prepare_losoto_bandpass.control.flagextend.operation = FLAGEXTEND prepare_losoto_bandpass.control.flagextend.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.flagextend.axesToExt = [time,freq] prepare_losoto_bandpass.control.flagextend.size = [200,80] prepare_losoto_bandpass.control.flagextend.percent = 50 prepare_losoto_bandpass.control.flagextend.maxCycles = 2 prepare_losoto_bandpass.control.plotA2.operation = PLOT prepare_losoto_bandpass.control.plotA2.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.plotA2.axesInPlot = [time,freq] prepare_losoto_bandpass.control.plotA2.axisInTable = ant prepare_losoto_bandpass.control.plotA2.plotFlag = True prepare_losoto_bandpass.control.plotA2.prefix = {{ inspection_directory }}/ampAFlag_ prepare_losoto_bandpass.control.merge.operation = REWEIGHT prepare_losoto_bandpass.control.merge.mode = copy prepare_losoto_bandpass.control.merge.soltab = sol000/phase000 prepare_losoto_bandpass.control.merge.soltabImport = amplitude000 prepare_losoto_bandpass.control.smooth.operation = SMOOTH prepare_losoto_bandpass.control.smooth.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.smooth.pol = XX,YY prepare_losoto_bandpass.control.smooth.axesToSmooth = [time] prepare_losoto_bandpass.control.smooth.mode = median prepare_losoto_bandpass.control.smooth.log = True prepare_losoto_bandpass.control.smooth.replace = False prepare_losoto_bandpass.control.bandpass.operation = SMOOTH prepare_losoto_bandpass.control.bandpass.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.bandpass.pol = XX,YY prepare_losoto_bandpass.control.bandpass.axesToSmooth = [freq] prepare_losoto_bandpass.control.bandpass.mode = savitzky-golay prepare_losoto_bandpass.control.bandpass.size = [17] prepare_losoto_bandpass.control.bandpass.degree = 2 prepare_losoto_bandpass.control.bandpass.log = True prepare_losoto_bandpass.control.interp.operation = INTERPOLATE prepare_losoto_bandpass.control.interp.soltab = sol000/amplitude000 prepare_losoto_bandpass.control.interp.outSoltab = bandpass prepare_losoto_bandpass.control.interp.axisToRegrid = freq prepare_losoto_bandpass.control.interp.maxFlaggedWidth = {{ max2interpolate }} prepare_losoto_bandpass.control.interp.newDelta = {{ bandpass_freqresolution }} prepare_losoto_bandpass.control.interp.delta = {{ avg_freqresolution }} prepare_losoto_bandpass.control.interp.log = True prepare_losoto_bandpass.control.smoothb.operation = SMOOTH prepare_losoto_bandpass.control.smoothb.soltab = sol000/bandpass prepare_losoto_bandpass.control.smoothb.pol = XX,YY prepare_losoto_bandpass.control.smoothb.axesToSmooth = [time] prepare_losoto_bandpass.control.smoothb.mode = median prepare_losoto_bandpass.control.smoothb.log = True prepare_losoto_bandpass.control.smoothb.replace = True prepare_losoto_bandpass.control.plotB1.operation = PLOT prepare_losoto_bandpass.control.plotB1.soltab = sol000/bandpass prepare_losoto_bandpass.control.plotB1.axesInPlot = [time,freq] prepare_losoto_bandpass.control.plotB1.axisInTable = ant prepare_losoto_bandpass.control.plotB1.plotFlag = True prepare_losoto_bandpass.control.plotB1.prefix = {{ inspection_directory }}/bandpass_ prepare_losoto_bandpass.control.plotB2.operation = PLOT prepare_losoto_bandpass.control.plotB2.soltab = sol000/bandpass prepare_losoto_bandpass.control.plotB2.axesInPlot = freq prepare_losoto_bandpass.control.plotB2.axisInTable = ant prepare_losoto_bandpass.control.plotB2.axisInCol = pol prepare_losoto_bandpass.control.plotB2.plotFlag = True prepare_losoto_bandpass.control.plotB2.prefix = {{ inspection_directory }}/bandpass_ prepare_losoto_bandpass.control.plotB2.time.minmaxstep = [0,1e20,500000] prepare_losoto_bandpass.control.plotB3.operation = PLOT prepare_losoto_bandpass.control.plotB3.soltab = sol000/bandpass prepare_losoto_bandpass.control.plotB3.axesInPlot = freq prepare_losoto_bandpass.control.plotB3.axisInCol = ant prepare_losoto_bandpass.control.plotB3.plotFlag = True prepare_losoto_bandpass.control.plotB3.prefix = {{ inspection_directory }}/bandpass_ prepare_losoto_bandpass.control.plotB3.time.minmaxstep = [0,1e20,500000] # create losoto v2 parset file prepare_losoto_bandpasstrans.control.kind = plugin prepare_losoto_bandpasstrans.control.type = makeLosotoParset prepare_losoto_bandpasstrans.control.steps = [plotB1,plotB2,plotB3] prepare_losoto_bandpasstrans.control.filename = {{ job_directory }}/losoto_transfer.parset prepare_losoto_bandpasstrans.control.global.ncpu = {{ num_proc_per_node }} prepare_losoto_bandpasstrans.control.plotB1.operation = PLOT prepare_losoto_bandpasstrans.control.plotB1.soltab = calibrator/bandpass prepare_losoto_bandpasstrans.control.plotB1.axesInPlot = [time,freq] prepare_losoto_bandpasstrans.control.plotB1.axisInTable = ant prepare_losoto_bandpasstrans.control.plotB1.plotFlag = True prepare_losoto_bandpasstrans.control.plotB1.prefix = {{ inspection_directory }}/bandpass_transfer_ prepare_losoto_bandpasstrans.control.plotB2.operation = PLOT prepare_losoto_bandpasstrans.control.plotB2.soltab = calibrator/bandpass prepare_losoto_bandpasstrans.control.plotB2.axesInPlot = freq prepare_losoto_bandpasstrans.control.plotB2.axisInTable = ant prepare_losoto_bandpasstrans.control.plotB2.axisInCol = pol prepare_losoto_bandpasstrans.control.plotB2.plotFlag = True prepare_losoto_bandpasstrans.control.plotB2.prefix = {{ inspection_directory }}/bandpass_transfer_ prepare_losoto_bandpasstrans.control.plotB2.time.minmaxstep = [0,1e20,500000] prepare_losoto_bandpasstrans.control.plotB3.operation = PLOT prepare_losoto_bandpasstrans.control.plotB3.soltab = calibrator/bandpass prepare_losoto_bandpasstrans.control.plotB3.axesInPlot = freq prepare_losoto_bandpasstrans.control.plotB3.axisInCol = ant prepare_losoto_bandpasstrans.control.plotB3.plotFlag = True prepare_losoto_bandpasstrans.control.plotB3.prefix = {{ inspection_directory }}/bandpass_transfer_ prepare_losoto_bandpasstrans.control.plotB3.time.minmaxstep = [0,1e20,500000] # do the processing on the LoSoTo file process_losoto_bandpass.control.kind = recipe process_losoto_bandpass.control.type = executable_args process_losoto_bandpass.control.inplace = True process_losoto_bandpass.control.executable = {{ losoto_directory }}/bin/losoto process_losoto_bandpass.control.max_per_node = {{ num_proc_per_node }} process_losoto_bandpass.control.mapfile_in = h5imp_cal_bandpass.output.mapfile process_losoto_bandpass.control.inputkey = h5in process_losoto_bandpass.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] # output the final soltab into an external h5parm h5exp_cal_bandpass.control.kind = recipe h5exp_cal_bandpass.control.type = executable_args h5exp_cal_bandpass.control.inplace = True h5exp_cal_bandpass.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5exp_cal_bandpass.control.error_tolerance = {{ error_tolerance }} h5exp_cal_bandpass.control.mapfile_in = h5imp_cal_bandpass.output.mapfile h5exp_cal_bandpass.control.inputkey = h5in h5exp_cal_bandpass.argument.flags = [-q,-v,h5in] h5exp_cal_bandpass.argument.insoltab = bandpass h5exp_cal_bandpass.argument.outh5parm = {{ cal_solutions }} h5exp_cal_bandpass.argument.outsolset = calibrator # transfer solutions if necessary transfer_solutions.control.type = pythonplugin transfer_solutions.control.executable = {{ scripts }}/transfer_solutions.py transfer_solutions.control.error_tolerance = {{ error_tolerance }} transfer_solutions.control.skip_infile = True transfer_solutions.control.mapfile_in = sky_cal.output.SkymodelName.mapfile transfer_solutions.argument.flags = [{{ cal_solutions }}, {{ solutions2transfer }}] transfer_solutions.argument.insolset = calibrator transfer_solutions.argument.outsolset = calibrator transfer_solutions.argument.insoltab = bandpass transfer_solutions.argument.outsoltab = bandpass transfer_solutions.argument.antenna = {{ antennas2transfer }} transfer_solutions.argument.parset = {{ job_directory }}/losoto_transfer.parset ################################ ## calibrate ionosphere ## ################################ # collect all instrument tables into one h5parm h5imp_cal_ion.control.kind = recipe h5imp_cal_ion.control.type = executable_args h5imp_cal_ion.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5imp_cal_ion.control.error_tolerance = {{ error_tolerance }} h5imp_cal_ion.control.mapfile_in = h5imp_cal_map.output.mapfile h5imp_cal_ion.control.inputkey = h5parm h5imp_cal_ion.control.outputkey = outh5parm h5imp_cal_ion.argument.flags = [-q,-v,-c,h5parm] h5imp_cal_ion.argument.outh5parm = outh5parm # create losoto v2 parset file prepare_losoto_ion.control.kind = plugin prepare_losoto_ion.control.type = makeLosotoParset prepare_losoto_ion.control.steps = [plotA1,flag,flagextend,plotA2,merge,plotP3,plotPd,duplicatePbkp,{{ cal_ion }},plotClock,plotPr,plotPr3,flagstation] prepare_losoto_ion.control.filename = {{ job_directory }}/losoto.parset prepare_losoto_ion.control.global.ncpu = {{ num_proc_per_node }} prepare_losoto_ion.control.plotA1.operation = PLOT prepare_losoto_ion.control.plotA1.soltab = sol000/amplitude000 prepare_losoto_ion.control.plotA1.axesInPlot = [time,freq] prepare_losoto_ion.control.plotA1.axisInTable = ant prepare_losoto_ion.control.plotA1.plotFlag = True prepare_losoto_ion.control.plotA1.prefix = {{ inspection_directory }}/ion_ampBFlag_ prepare_losoto_ion.control.flag.operation = FLAG prepare_losoto_ion.control.flag.soltab = sol000/amplitude000 prepare_losoto_ion.control.flag.axesToFlag = [time,freq] prepare_losoto_ion.control.flag.order = [100,40] prepare_losoto_ion.control.flag.maxCycles = 1 prepare_losoto_ion.control.flag.maxRms = 5 prepare_losoto_ion.control.flag.replace = False prepare_losoto_ion.control.flag.preFlagZeros = False prepare_losoto_ion.control.flag.mode = smooth prepare_losoto_ion.control.flagextend.operation = FLAGEXTEND prepare_losoto_ion.control.flagextend.soltab = sol000/amplitude000 prepare_losoto_ion.control.flagextend.axesToExt = [time,freq] prepare_losoto_ion.control.flagextend.size = [200,80] prepare_losoto_ion.control.flagextend.percent = 50 prepare_losoto_ion.control.flagextend.maxCycles = 2 prepare_losoto_ion.control.plotA2.operation = PLOT prepare_losoto_ion.control.plotA2.soltab = sol000/amplitude000 prepare_losoto_ion.control.plotA2.axesInPlot = [time,freq] prepare_losoto_ion.control.plotA2.axisInTable = ant prepare_losoto_ion.control.plotA2.plotFlag = True prepare_losoto_ion.control.plotA2.prefix = {{ inspection_directory }}/ion_ampAFlag_ prepare_losoto_ion.control.merge.operation = REWEIGHT prepare_losoto_ion.control.merge.mode = copy prepare_losoto_ion.control.merge.soltab = sol000/phase000 prepare_losoto_ion.control.merge.soltabImport = amplitude000 prepare_losoto_ion.control.plotP3.operation = PLOT prepare_losoto_ion.control.plotP3.soltab = sol000/phase000 prepare_losoto_ion.control.plotP3.axesInPlot = [time,freq] prepare_losoto_ion.control.plotP3.axisInTable = ant prepare_losoto_ion.control.plotP3.plotFlag = True prepare_losoto_ion.control.plotP3.prefix = {{ inspection_directory }}/ion_ph_ prepare_losoto_ion.control.plotP3.refAnt = {{ refant }} prepare_losoto_ion.control.plotP3.minmax = [-3.14,3.14] prepare_losoto_ion.control.plotPd.operation = PLOT prepare_losoto_ion.control.plotPd.soltab = sol000/phase000 prepare_losoto_ion.control.plotPd.axesInPlot = [time,freq] prepare_losoto_ion.control.plotPd.axisInTable = ant prepare_losoto_ion.control.plotPd.axisDiff = pol prepare_losoto_ion.control.plotPd.plotFlag = True prepare_losoto_ion.control.plotPd.prefix = {{ inspection_directory }}/ion_ph_poldif prepare_losoto_ion.control.plotPd.refAnt = {{ refant }} prepare_losoto_ion.control.plotPd.minmax = [-3.14,3.14] prepare_losoto_ion.control.duplicatePbkp.operation = DUPLICATE prepare_losoto_ion.control.duplicatePbkp.soltab = sol000/phase000 prepare_losoto_ion.control.duplicatePbkp.soltabOut = phaseOrig prepare_losoto_ion.control.ct.operation = CLOCKTEC prepare_losoto_ion.control.ct.soltab = sol000/phase000 prepare_losoto_ion.control.ct.clocksoltabOut = clock prepare_losoto_ion.control.ct.tecsoltabOut = tec prepare_losoto_ion.control.ct.tec3rdsoltabOut = tec3rd prepare_losoto_ion.control.ct.offsetsoltabOut = phase_offset prepare_losoto_ion.control.ct.CombinePol = True prepare_losoto_ion.control.ct.FlagBadChannels = False prepare_losoto_ion.control.ct.Fit3rdOrder = False prepare_losoto_ion.control.ct.Circular = False prepare_losoto_ion.control.ct3.operation = CLOCKTEC prepare_losoto_ion.control.ct3.soltab = sol000/phase000 prepare_losoto_ion.control.ct3.clocksoltabOut = clock prepare_losoto_ion.control.ct3.tecsoltabOut = tec prepare_losoto_ion.control.ct3.tec3rdsoltabOut = tec3rd prepare_losoto_ion.control.ct3.offsetsoltabOut = phase_offset prepare_losoto_ion.control.ct3.CombinePol = True prepare_losoto_ion.control.ct3.FlagBadChannels = False prepare_losoto_ion.control.ct3.Fit3rdOrder = True prepare_losoto_ion.control.ct3.Circular = False prepare_losoto_ion.control.plotClock.operation = PLOT prepare_losoto_ion.control.plotClock.soltab = sol000/clock prepare_losoto_ion.control.plotClock.axesInPlot = [time] prepare_losoto_ion.control.plotClock.axisInTable = ant prepare_losoto_ion.control.plotClock.prefix = {{ inspection_directory }}/clock prepare_losoto_ion.control.plotClock.plotFlag = False prepare_losoto_ion.control.plotClock.refAnt = {{ refant }} prepare_losoto_ion.control.plotTEC.operation = PLOT prepare_losoto_ion.control.plotTEC.soltab = sol000/tec prepare_losoto_ion.control.plotTEC.axesInPlot = [time] prepare_losoto_ion.control.plotTEC.axisInTable = ant prepare_losoto_ion.control.plotTEC.prefix = {{ inspection_directory }}/tec prepare_losoto_ion.control.plotTEC.plotFlag = False prepare_losoto_ion.control.plotTEC.refAnt = {{ refant }} prepare_losoto_ion.control.plotTEC3.operation = PLOT prepare_losoto_ion.control.plotTEC3.soltab = sol000/tec3rd prepare_losoto_ion.control.plotTEC3.axesInPlot = [time] prepare_losoto_ion.control.plotTEC3.axisInTable = ant prepare_losoto_ion.control.plotTEC3.prefix = {{ inspection_directory }}/tec3rd prepare_losoto_ion.control.plotTEC3.plotFlag = False prepare_losoto_ion.control.plotTEC3.refAnt = {{ refant }} prepare_losoto_ion.control.smooth.operation = SMOOTH prepare_losoto_ion.control.smooth.soltab = sol000/clock prepare_losoto_ion.control.smooth.axesToSmooth = [time] prepare_losoto_ion.control.smooth.mode = median prepare_losoto_ion.control.smooth.log = False prepare_losoto_ion.control.smooth.replace = True prepare_losoto_ion.control.residuals.operation = RESIDUALS prepare_losoto_ion.control.residuals.soltab = sol000/phase000 prepare_losoto_ion.control.residuals.soltabsToSub = [tec,clock] prepare_losoto_ion.control.residuals3.operation = RESIDUALS prepare_losoto_ion.control.residuals3.soltab = sol000/phase000 prepare_losoto_ion.control.residuals3.soltabsToSub = [tec,clock,tec3rd] # only for very low-freq dataset prepare_losoto_ion.control.plotPr.operation = PLOT prepare_losoto_ion.control.plotPr.soltab = sol000/phase000 prepare_losoto_ion.control.plotPr.axesInPlot = [time,freq] prepare_losoto_ion.control.plotPr.axisInTable = ant prepare_losoto_ion.control.plotPr.axisDiff = pol prepare_losoto_ion.control.plotPr.plotFlag = True prepare_losoto_ion.control.plotPr.prefix = {{ inspection_directory }}/ion_ph-res_poldif prepare_losoto_ion.control.plotPr.refAnt = {{ refant }} prepare_losoto_ion.control.plotPr.minmax = [-3.14,3.14] prepare_losoto_ion.control.plotPr3.operation = PLOT prepare_losoto_ion.control.plotPr3.soltab = sol000/phase000 prepare_losoto_ion.control.plotPr3.axesInPlot = [time,freq] prepare_losoto_ion.control.plotPr3.axisInTable = ant prepare_losoto_ion.control.plotPr3.plotFlag = True prepare_losoto_ion.control.plotPr3.prefix = {{ inspection_directory }}/ion_ph-res_ prepare_losoto_ion.control.plotPr3.refAnt = {{ refant }} prepare_losoto_ion.control.plotPr3.minmax = [-3.14,3.14] prepare_losoto_ion.control.flagstation.operation = FLAGSTATION prepare_losoto_ion.control.flagstation.soltab = sol000/phase000 prepare_losoto_ion.control.flagstation.maxStddev = {{ maxStddev }} prepare_losoto_ion.control.flagstation.mode = resid prepare_losoto_ion.control.flagstation.refAnt = {{ refant }} prepare_losoto_ion.control.flagstation.soltabExport = clock # do the processing on the LoSoTo file process_losoto_ion.control.kind = recipe process_losoto_ion.control.type = executable_args process_losoto_ion.control.inplace = True process_losoto_ion.control.executable = {{ losoto_directory }}/bin/losoto process_losoto_ion.control.max_per_node = {{ num_proc_per_node }} process_losoto_ion.control.mapfile_in = h5imp_cal_ion.output.mapfile process_losoto_ion.control.inputkey = h5in process_losoto_ion.argument.flags = [-v,h5in,{{ job_directory }}/losoto.parset] # output the final soltab into an external h5parm h5exp_cal_ion.control.kind = recipe h5exp_cal_ion.control.type = executable_args h5exp_cal_ion.control.inplace = True h5exp_cal_ion.control.executable = {{ losoto_directory }}/bin/H5parm_collector.py h5exp_cal_ion.control.error_tolerance = {{ error_tolerance }} h5exp_cal_ion.control.mapfile_in = h5imp_cal_ion.output.mapfile h5exp_cal_ion.control.inputkey = h5in h5exp_cal_ion.argument.flags = [-q,-v,h5in] h5exp_cal_ion.argument.insoltab = {{ tables2export }} h5exp_cal_ion.argument.outh5parm = {{ cal_solutions }} h5exp_cal_ion.argument.outsolset = calibrator ################################ ## applying the results ## ################################ # apply the PA solutions apply_PA.control.type = dppp apply_PA.control.error_tolerance = {{ error_tolerance }} apply_PA.control.inplace = True apply_PA.control.max_per_node = {{ num_proc_per_node_limit }} apply_PA.control.mapfile_in = ndppp_prep_cal.output.mapfile apply_PA.control.inputkey = msfile apply_PA.argument.msin = msfile apply_PA.argument.numthreads = {{ max_dppp_threads }} apply_PA.argument.msin.datacolumn = DATA apply_PA.argument.msout.datacolumn = CORRECTED_DATA apply_PA.argument.msout.storagemanager = "Dysco" apply_PA.argument.msout.storagemanager.databitrate = 0 apply_PA.argument.steps = [applyPA] apply_PA.argument.applyPA.type = applycal apply_PA.argument.applyPA.correction = polalign apply_PA.argument.applyPA.parmdb = {{ cal_solutions }} # apply the bandpass apply_bandpass.control.type = dppp apply_bandpass.control.error_tolerance = {{ error_tolerance }} apply_bandpass.control.inplace = True apply_bandpass.control.max_per_node = {{ num_proc_per_node_limit }} apply_bandpass.control.mapfile_in = ndppp_prep_cal.output.mapfile apply_bandpass.control.inputkey = msfile apply_bandpass.argument.msin = msfile apply_bandpass.argument.numthreads = {{ max_dppp_threads }} apply_bandpass.argument.msin.datacolumn = CORRECTED_DATA apply_bandpass.argument.msout.datacolumn = CORRECTED_DATA apply_bandpass.argument.msout.storagemanager = "Dysco" apply_bandpass.argument.msout.storagemanager.databitrate = 0 apply_bandpass.argument.steps = [applybandpass] apply_bandpass.argument.applybandpass.type = applycal apply_bandpass.argument.applybandpass.correction = bandpass apply_bandpass.argument.applybandpass.parmdb = {{ cal_solutions }} apply_bandpass.argument.applybandpass.updateweights = True # apply the beam apply_beam.control.type = dppp apply_beam.control.error_tolerance = {{ error_tolerance }} apply_beam.control.inplace = True apply_beam.control.max_per_node = {{ num_proc_per_node_limit }} apply_beam.control.mapfile_in = ndppp_prep_cal.output.mapfile apply_beam.control.inputkey = msfile apply_beam.argument.msin = msfile apply_beam.argument.numthreads = {{ max_dppp_threads }} apply_beam.argument.msin.datacolumn = CORRECTED_DATA apply_beam.argument.msout.datacolumn = CORRECTED_DATA apply_beam.argument.msout.storagemanager = "Dysco" apply_beam.argument.msout.storagemanager.databitrate = 0 apply_beam.argument.steps = [applybeam] apply_beam.argument.applybeam.type = applybeam apply_beam.argument.applybeam.invert = True apply_beam.argument.applybeam.usechannelfreq = False apply_beam.argument.applybeam.beammode = element apply_beam.argument.applybeam.updateweights = True # apply the FR solutions apply_FR.control.type = dppp apply_FR.control.error_tolerance = {{ error_tolerance }} apply_FR.control.inplace = True apply_FR.control.max_per_node = {{ num_proc_per_node_limit }} apply_FR.control.mapfile_in = ndppp_prep_cal.output.mapfile apply_FR.control.inputkey = msfile apply_FR.argument.msin = msfile apply_FR.argument.numthreads = {{ max_dppp_threads }} apply_FR.argument.msin.datacolumn = CORRECTED_DATA apply_FR.argument.msout.datacolumn = CORRECTED_DATA apply_FR.argument.msout.storagemanager = "Dysco" apply_FR.argument.msout.storagemanager.databitrate = 0 apply_FR.argument.steps = [applyFR] apply_FR.argument.applyFR.type = applycal apply_FR.argument.applyFR.correction = faraday apply_FR.argument.applyFR.parmdb = {{ cal_solutions }} # apply the clock solutions apply_clock.control.type = dppp apply_clock.control.error_tolerance = {{ error_tolerance }} apply_clock.control.inplace = True apply_clock.control.max_per_node = {{ num_proc_per_node_limit }} apply_clock.control.mapfile_in = ndppp_prep_cal.output.mapfile apply_clock.control.inputkey = msfile apply_clock.argument.msin = msfile apply_clock.argument.numthreads = {{ max_dppp_threads }} apply_clock.argument.msin.datacolumn = CORRECTED_DATA apply_clock.argument.msout.datacolumn = CORRECTED_DATA apply_clock.argument.msout.storagemanager = "Dysco" apply_clock.argument.msout.storagemanager.databitrate = 0 apply_clock.argument.steps = [applyclock] apply_clock.argument.applyclock.type = applycal apply_clock.argument.applyclock.correction = clock apply_clock.argument.applyclock.parmdb = {{ cal_solutions }} # apply the TEC solutions apply_TEC.control.type = dppp apply_TEC.control.error_tolerance = {{ error_tolerance }} apply_TEC.control.inplace = True apply_TEC.control.max_per_node = {{ num_proc_per_node_limit }} apply_TEC.control.mapfile_in = ndppp_prep_cal.output.mapfile apply_TEC.control.inputkey = msfile apply_TEC.argument.msin = msfile apply_TEC.argument.numthreads = {{ max_dppp_threads }} apply_TEC.argument.msin.datacolumn = CORRECTED_DATA apply_TEC.argument.msout.datacolumn = CORRECTED_DATA apply_TEC.argument.msout.storagemanager = "Dysco" apply_TEC.argument.msout.storagemanager.databitrate = 0 apply_TEC.argument.steps = [applytec] apply_TEC.argument.applytec.type = applycal apply_TEC.argument.applytec.correction = tec apply_TEC.argument.applytec.parmdb = {{ cal_solutions }} # # apply the phase offset solutions # apply_offset.control.type = dppp # apply_offset.control.error_tolerance = {{ error_tolerance }} # apply_offset.control.inplace = True # apply_offset.control.max_per_node = {{ num_proc_per_node_limit }} # apply_offset.control.mapfile_in = ndppp_prep_cal.output.mapfile # apply_offset.control.inputkey = msfile # apply_offset.argument.msin = msfile # apply_offset.argument.numthreads = {{ max_dppp_threads }} # apply_offset.argument.msin.datacolumn = CORRECTED_DATA # apply_offset.argument.msout.datacolumn = CORRECTED_DATA # apply_offset.argument.msout.storagemanager = "Dysco" # apply_offset.argument.msout.storagemanager.databitrate = 0 # apply_offset.argument.steps = [applyoffset] # apply_offset.argument.applyoffset.type = applycal # apply_offset.argument.applyoffset.correction = phase_offset # apply_offset.argument.applyoffset.parmdb = {{ cal_solutions }} ############################### ## finalizing the results ## ################################ # set the pointing direction h5parm_name.control.type = pythonplugin h5parm_name.control.executable = {{ scripts }}/h5parm_pointingname.py h5parm_name.control.error_tolerance = {{ error_tolerance }} h5parm_name.argument.flags = [{{ cal_solutions }}] h5parm_name.argument.solsetName = calibrator h5parm_name.argument.pointing = sky_cal.output.SkymodelName.mapfile # set the pointing direction make_summary.control.type = pythonplugin make_summary.control.executable = {{ scripts }}/make_summary.py make_summary.control.error_tolerance = {{ error_tolerance }} make_summary.control.mapfile_in = combine_data_cal_map.output.mapfile make_summary.control.inputkey = infiles make_summary.argument.observation_directory = {{ working_directory }} make_summary.argument.logfile = {{ log_file }} make_summary.argument.h5parmdb = {{ cal_solutions }} make_summary.argument.inspection_directory = {{ inspection_directory }} make_summary.argument.MSfile = infiles ######################################################## ## ## ## END PIPELINE ## ## ## ########################################################