Newer
Older
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
##########################################################################
# Pre-Facet Calibrator Calibration Pipeline v3.0 (04/09/2019) #
# #
# Calibrator part of the basic Pre-Facet calibration pipeline: #
# - requires LOFAR software version >= 3.1.0 #
# - requires losoto software version >= 2.0.0 #
# - expects shared filesystem, that all nodes can reach all files! #
# (E.g. a single workstation or compute cluster with shared filesystem #
# doesn't work on multiple nodes on CEP3.) #
##########################################################################
##########################################
### parameters you will need to adjust. ##
##########################################
## information about the calibrator data
! cal_input_path = /input_data/calib ## specify the directory where your calibrator data is stored
! cal_input_pattern = *.MS ## regular expression pattern of all your calibrator files
## location of the software
! prefactor_directory = /opt/prefactor/ ## path to your prefactor copy
! losoto_directory = /opt/lofarsoft ## path to your local LoSoTo installation
! aoflagger = /opt/lofarsoft/bin/aoflagger ## path to your aoflagger executable
##########################################
### parameters you may need to adjust ##
##########################################
! refant = 'CS001HBA0' ## name of the station that will be used as a reference for the phase-plots, 'closest' will reference to the spatially closest unflagged antenna
! flag_baselines = [] ## NDPPP-compatible pattern for baselines or stations to be flagged (may be an empty list, i.e.: [] )
! process_baselines_cal = *& ## performs A-Team-clipping/demixing and direction-independent phase-only self-calibration only on these baselines. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
! filter_baselines = {{ process_baselines_cal }} ## selects only this set of baselines to be processed. Choose [CR]S*& if you want to process only cross-correlations and remove international stations.
! do_smooth = False ## enable or disable baseline-based smoothing
! rfistrategy = HBAdefault.rfis ## strategy to be applied with the statistical flagger (AOFlagger)
! max2interpolate = 30 ## amount of channels in which interpolation should be performed for deriving the bandpass
! ampRange = [0, 0] ## range of median amplitudes accepted per station. Use [0, 0] to use typical ranges of [50, 225] for the current correlator or [0.0004, 0.0018] for the old one.
! skip_international = True ## skip fitting the bandpass for international stations (this avoids flagging them in many cases)
! raw_data = False ## use autoweight, set to True in case you are using raw data
! propagatesolutions = True ## use already derived solutions as initial guess for the upcoming time slot
! flagunconverged = False ## flag solutions for solves that did not converge (if they were also detected to diverge)
! maxStddev = -1.0 ## Maximum allowable standard deviation when outlier clipping is done. For phases, this should value should be in radians, for amplitudes in log(amp). If None (or negative), a value of 0.1 rad is used for phases and 0.01 for amplitudes
# options for solution transfer (for non-trusted calibrator observations)
! solutions2transfer = {{ prefactor_directory }}/solutions/3C48.h5 ## location of h5parm reference solutions to transfer
! antennas2transfer = [FUSPID].* ## regular expression of antennas which solutions should be transferred to the final solution set, if none of the trusted calibrators have been observed (by default: all international stations)
# demixing options (only used if demix step is added to the prep_cal_strategy variable)
! demix_sources = [CasA,CygA] ## choose sources to demix (provided as list)
! demix_target = "" ## if given, the target source model (its patch in the SourceDB) is taken into account when solving
! demix_freqstep = 16 ## number of channels to average when demixing.
! demix_timestep = 10 ## number of time slots to average when demixing
# definitions for pipeline options -- do not change!
! default_flagging = flagbaseline,flagelev,flagamp ## regular flagging after pre-processing by the observatory pipelines
! raw_flagging = flagedge,aoflag,{{ default_flagging }} ## full flagging (usually only necessary for raw data)
! 1st_order = ct,plotTEC,residuals ## Do not change! Only cal_ion should be edited if needed
! 3rd_order = ct3,plotTEC,plotTEC3,residuals3 ## Do not change! Only cal_ion should be edited if needed
! demix = demix, ## Do not change! Only demix_step should be edited if needed
! full_apply = ,apply_PA,apply_bandpass,apply_clock,apply_beam,apply_FR,apply_TEC ## Do not change!
! none = ## Do not change!
# pipeline options
! initial_flagging = {{ default_flagging }} ## choose {{ raw_flagging }} if you process raw data
! demix_step = {{ none }} ## choose {{ demix }} if you want to demix
! cal_ion = {{ 1st_order }},smooth ## choose {{ 3rd_order }} if you want to include 3rd order ionospheric effects (may be useful for LBA < 35 MHz), add smooth if you want to use the median of the clock in time (suggested for HBA+LB)
! tables2export = clock ## comma-separated list of tables to export from the ionospheric calibration step (cal_ion)
! final_apply = {{ none }} ## choose {{ full_apply }} if you want to apply all extracted solutions to the calibrator field
##########################################
### parameters for pipeline performance ##
##########################################
! num_proc_per_node = input.output.max_per_node ## number of processes to use per step per node (usually max_per_node from pipeline.cfg)
! num_proc_per_node_limit = 4 ## number of processes to use per step per node for tasks with high i/o (dppp or cp) or memory (eg calibration)
! max_dppp_threads = 10 ## number of threads per process for NDPPP
! memoryperc = 20 ## maximum of memory used for aoflagger in raw_flagging mode in percent
! min_length = 50 ## minimum amount of subbands to concatenate in frequency necessary to perform the wide-band flagging in the RAM. It data is too big aoflag will use indirect-read.
! overhead = 0.8 ## Only use this fraction of the available memory for deriving the amount of data to be concatenated.
! min_separation = 30 ## minimal accepted distance to an A-team source on the sky in degrees (will raise a WARNING)
! max_separation_arcmin = 1.0 ## maximum distance to the phase center for which a skymodel will be still accepted. Change with caution!
! error_tolerance = False ## set this to True if you want the pipeline run to continue if single bands fail
##########################################
### parameters you may want to adjust ##
##########################################
## main directories
! lofar_directory = $LOFARROOT ## base directory of your LOFAR installation
! job_directory = input.output.job_directory ## directory of the prefactor outputs
! working_directory = input.output.working_directory/input.output.job_name ## specify the working_directory (intermediate data products)
! log_file = input.output.log_file ## location of the logfile
! mapfile_dir = input.output.mapfile_dir ## specify mapfile directory
## script and plugin directories
! scripts = {{ prefactor_directory }}/scripts
pipeline.pluginpath = {{ prefactor_directory }}/plugins
## skymodel directory
! calibrator_path_skymodel = {{ prefactor_directory }}/skymodels
! A-team_skymodel = {{ calibrator_path_skymodel }}/Ateam_LBA_CC.skymodel
## result directories
! results_directory = {{ job_directory }}/results ## location of the results
! inspection_directory = {{ results_directory }}/inspection ## directory where the inspection plots will be stored
! cal_values_directory = {{ results_directory }}/cal_values ## directory where the final h5parm solution set will be stored
## calibrator solutions
! cal_solutions = {{ cal_values_directory }}/cal_solutions.h5
## averaging for the calibrator data
! avg_timeresolution = 4. ## average to 4 sec/timeslot
! avg_freqresolution = 48.82kHz ## average to 48.82 kHz/ch (= 4 ch/SB)
! bandpass_freqresolution = 195.3125kHz ## resolution of the bandpass table is 195.3125kHz (= 1 ch/SB)
########################################################
## ##
## BEGIN PIPELINE: DO NOT UPDATE BELOW THIS LINE! ##
## ##
########################################################
# which steps to run
pipeline.steps = [prep, PA, FR, bandpass, ion, finalize]
# pipeline substeps
pipeline.steps.prep = [createmap_cal, combine_data_map, check_Ateam_separation, mk_cal_values_dir, createmap_prepcal, createmap_instcal, create_ateam_model_map, make_sourcedb_ateam, expand_sourcedb_ateam, ndppp_prep_cal, combine_data_cal_map, ms_concat, ms_concat_map, expand_memory_map, aoflag, sky_cal, make_sourcedb, expand_sourcedb, expand_skymodel, calib_cal_parmmap, h5imp_cal_map, smooth_data, predict_cal]
pipeline.steps.PA = [calib_cal, h5imp_cal_PA, prepare_losoto_PA, process_losoto_PA, h5exp_cal_PA, apply_PA, apply_beam]
pipeline.steps.FR = [smooth_corrected, calib_cal, h5imp_cal_FR, prepare_losoto_FR, process_losoto_FR, h5exp_cal_FR, apply_FR]
pipeline.steps.bandpass = [smooth_corrected, calib_cal2, h5imp_cal_bandpass, prepare_losoto_bandpass, prepare_losoto_bandpasstrans, process_losoto_bandpass, h5exp_cal_bandpass, transfer_solutions, apply_PA, apply_bandpass, apply_beam, apply_FR]
pipeline.steps.ion = [smooth_corrected, calib_cal2, h5imp_cal_ion, prepare_losoto_ion, process_losoto_ion, h5exp_cal_ion]
pipeline.steps.finalize = [h5parm_name {{ final_apply }}, make_summary]
###############################
## Mapping calibrator files ##
###############################
# generate a mapfile of all the calibrator data
createmap_cal.control.kind = plugin
createmap_cal.control.type = createMapfile
createmap_cal.control.method = mapfile_from_folder
createmap_cal.control.mapfile_dir = {{ mapfile_dir }}
createmap_cal.control.filename = createmap_cal.mapfile
createmap_cal.control.folder = {{ cal_input_path }}
createmap_cal.control.pattern = {{ cal_input_pattern }}
# combine all entries into one mapfile, for the sortmap script
combine_data_map.control.kind = plugin
combine_data_map.control.type = createMapfile
combine_data_map.control.method = mapfile_all_to_one
combine_data_map.control.mapfile_dir = {{ mapfile_dir }}
combine_data_map.control.filename = combine_data_map.mapfile
combine_data_map.control.mapfile_in = createmap_cal.output.mapfile
# warn for potential nearby A-Team sources
check_Ateam_separation.control.type = pythonplugin
check_Ateam_separation.control.executable = {{ scripts }}/check_Ateam_separation.py
check_Ateam_separation.control.mapfile_in = combine_data_map.output.mapfile
check_Ateam_separation.control.inputkey = MSfile
check_Ateam_separation.argument.min_separation = {{ min_separation }}
check_Ateam_separation.argument.outputimage = {{ inspection_directory }}/A-Team_elevation_calibrator.png
check_Ateam_separation.argument.flags = [MSfile]
#############################
## Prepare for demixing ##
#############################
# generate a mapfile of the calibrator
createmap_prepcal.control.kind = plugin
createmap_prepcal.control.type = makeResultsMapfile
createmap_prepcal.control.mapfile_dir = {{ mapfile_dir }}
createmap_prepcal.control.filename = createmap_prepcal.mapfile
createmap_prepcal.control.mapfile_in = createmap_cal.output.mapfile
createmap_prepcal.control.target_dir = {{ job_directory }}
createmap_prepcal.control.make_target_dir = False
createmap_prepcal.control.new_suffix = .ndppp_prep_cal
# generate a mapfile for the instrument table of the calibrator
createmap_instcal.control.kind = plugin
createmap_instcal.control.type = changeMapfile
createmap_instcal.control.mapfile_in = createmap_prepcal.output.mapfile
createmap_instcal.control.join_files = instrument
createmap_instcal.control.newname = createmap_instcal.mapfile
# create a mapfile with the A-Team skymodel, length = 1
create_ateam_model_map.control.kind = plugin
create_ateam_model_map.control.type = addListMapfile
create_ateam_model_map.control.hosts = ['localhost']
create_ateam_model_map.control.files = [ {{ A-team_skymodel }} ]
create_ateam_model_map.control.mapfile_dir = {{ mapfile_dir }}
create_ateam_model_map.control.filename = ateam_model_name.mapfile
# make sourcedbs from the A-Team skymodel, length = 1
make_sourcedb_ateam.control.kind = recipe
make_sourcedb_ateam.control.type = executable_args
make_sourcedb_ateam.control.executable = {{ lofar_directory }}/bin/makesourcedb
make_sourcedb_ateam.control.error_tolerance = {{ error_tolerance }}
make_sourcedb_ateam.control.args_format = lofar
make_sourcedb_ateam.control.outputkey = out
make_sourcedb_ateam.control.mapfile_in = create_ateam_model_map.output.mapfile
make_sourcedb_ateam.control.inputkey = in
Loading full blame...