Newer
Older
import argparse
import os
import pandas as pd
import numpy as np
import yaml
from operator import attrgetter
from astropy.coordinates import SkyCoord
from glob import glob
from argparse import HelpFormatter
class SortingHelpFormatter(HelpFormatter):
def add_arguments(self, actions):
actions = sorted(actions, key=attrgetter('option_strings'))
super(SortingHelpFormatter, self).add_arguments(actions)
def parse_arguments(description, add_dl2=False, add_irf=False, add_job=False, add_dl3=False, create_runlist=False, add_common=False):
parser = argparse.ArgumentParser(description=description, formatter_class=SortingHelpFormatter)
# parser.add_argument('--config-analysis',
# type=str, default=None, dest='config_analysis',
# help='Specify a config file which describes analysis profile to use')
parser.add_argument('--verbose', '-v',
action='store_true', dest='verbose',
help='Increase output verbosity')
if create_runlist:
parser.add_argument('--source_name', '-n', required=True, default=None,
dest='source_name', type=str,
help='Name of the source')
parser.add_argument('--ra', type=float, dest='ra',
help='RA coordinate of the target. To add if you want to use custom position')
parser.add_argument('--dec', type=float, dest='dec',
help='Dec coordinate of the target. To add if you want to use custom position')
parser.add_argument('--night', type=str, dest='night',
nargs='+', required=False,
help='Night of the observation in the format YYYYMMDD. More nights can be specified')
if add_common:
parser.add_argument('--source_name', '-n', required=True,
default=None, dest='source_name', type=str,
help='Name of the source')
parser.add_argument('--config', '-c', type=str, default=None,
required=True, dest='config', help='Specify a personal config file for the analysis')
parser.add_argument('--dry',
default=False, required=False, action='store_true', dest='dry',
help='Make a dry run, no true submission')
if add_dl2:
parser.add_argument('--outdir', '-o', dest='outdir', required=False,
type=str, default=None, help='Directory to store the output')
parser.add_argument('--tcuname',
default=None, dest='tcuname', type=str,
help='Apply run selection based on TCU source name')
parser.add_argument('--runlist', '-rl',
default=None, dest='runlist', type=str,
help='File with a list of run and the associated night to be analysed')
parser.add_argument('--distance', '-dis',
type=float, dest='distance', default=-1,
help='Max distance in degrees between the target position and the run pointing position for the run selection. Negative value means no selection using this parameter (default: %(default)s).')
parser.add_argument('--ra',
type=float, dest='ra', default=-1,
help='RA coordinate of the target. To add if you want to use custom position')
parser.add_argument('--dec',
type=float, dest='dec', default=-91,
help='Dec coordinate of the target. To add if you want to use custom position')
if add_irf:
parser.add_argument('--gh_cut',
required=False, type=float, dest='gh_cut',
help='Fixed selection cut for gh_score (gammaness)')
parser.add_argument('--theta_cut',
required=False, type=float, dest='theta_cut',
help='Fixed selection cut for theta')
parser.add_argument('--obs_time',
required=False, type=float, dest='obs_time',
help='Observation time for IRF in hours')
if add_dl3:
parser.add_argument('--outdir', '-o', dest='outdir', type=str,
default=None, help='Directory to store the output')
parser.add_argument('--runlist', '-rl',
default=None, dest='runlist', type=str,
help='File with a list of run and the associated night to be analysed')
parser.add_argument('--ra',
type=float, dest='ra', default=-1,
help='RA coordinate of the target. To add if you want to use custom position')
parser.add_argument('--dec',
type=float, dest='dec', default=-91,
help='Dec coordinate of the target. To add if you want to use custom position')
parser.add_argument('--cut_file', '-cut',
default=None, dest='cut_file', type=str,
help='Cut file')
parser.add_argument('--gh_cut',
required=False, type=float, dest='gh_cut',
help='Fixed selection cut for gh_score (gammaness)')
parser.add_argument('--theta_cut',
required=False, type=float, dest='theta_cut',
help='Fixed selection cut for theta')
if add_job:
parser.add_argument('--submit', default=False, dest='submit',
action='store_true', required=False, help='Submit the cmd to slurm on site')
parser.add_argument('--globber', '-g',
dest='globber', action='store_true', required=False, default=False,
help='If True, overwrites existing output file without asking (default: %(default)s).')
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
args = parser.parse_args()
return args
def get_db(database_filename):
"""
Parameters
----------
database_filename : name of database file
---------
Returns the DB
"""
db_file = os.environ.get('CONFIG_FOLDER') + '/' + database_filename
database = pd.read_csv(db_file, index_col=0, parse_dates=True)
return database
def get_config(config_filename):
"""
Parameters
----------
config_filename : name of the configuration file
---------
Returns the configuration file
"""
config_file = os.environ.get('CONFIG_FOLDER') + '/' + config_filename
with open(config_file) as f:
config_analysis = yaml.load(f, Loader=yaml.FullLoader)
return config_analysis
def get_runs_database(args, database):
# make the run list into a np array
databaseRuns = np.array(database.index)
# Apply selection of data if argument is specified. Based on tcuname, run or coordinates
selection = database
#if args.tcuname[0] == 'all' and args.night[0] == 'all' and args.runlist[0] == 'none':
# raise RuntimeError("Cannont make a run selection. Either tcuname, night or runlist or distance is needed")
if args.tcuname is not None:
selection = database.loc[database['Target']].isin([args.tcuname])
if args.verbose:
print("Selection of runs based on the TCU name", args.tcuname, ". Only runs with the name in the TCU are kept")
print(selection.index)
# if args.night[0] != 'all':
# selection = database.loc[database['day'].isin(args.night)]
# if args.verbose:
# print("selection of night", args.night[0])
# print(selection.index)
if args.distance > 0:
if args.ra == -1 or args.dec == -91:
raise RuntimeError("Cannont make a run selection. Ra and Dec value of the source are not given.")
selection = database.loc[((database['RA_Obs'] - args.ra)**2 + (database['Dec_Obs'] - args.dec)**2) < args.distance**2]
if args.verbose:
print("Selection based on angular distance ", args.distance, "°")
print(selection.index)
databaseRuns = np.array(selection.index)
if args.runlist is not None:
rl = np.loadtxt(os.environ.get('CONFIG_FOLDER') + '/' + args.runlist, unpack=True, dtype=int)
#databaseRuns = np.array([a for a in rl if a in databaseRuns])
databaseRuns = np.array([a for a in rl])
if args.verbose:
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
return databaseRuns
def create_DL1_list(args, config, runs, night):
"""
create list with all the DL1 run path to analyze
Parameters
----------
runs: List of run numbers
night: night to analyze
"""
version = config['dl1_data']['version']
cleaning = config['dl1_data']['cleaning']
folder = config['data_folder'] + '/DL1/' + night + '/' + version + '/' + cleaning + '/dl1*' # Current format of LST data path
if args.verbose:
print("Looking for files here :", folder)
filepath_glob = glob(folder)
# initializa data frame
filelist = pd.DataFrame(columns=['path', 'night'])
# Create a list of files with matching run numbers
newEntry = {}
for filename in filepath_glob:
for run in runs:
if (f"Run{run:>05}.h5" in filename):
newEntry['path'] = filename
newEntry['night'] = night
filelist.loc[run] = newEntry['path'], newEntry['night']
return filelist
def create_DL2_list(args, config, runs, night):
"""
create list with all the DL2 run path to analyze
Parameters
----------
runs: List of run numbers
"""
version = config['dl2_data']['version']
cleaning = config['dl2_data']['cleaning']
folder = config['data_folder'] + '/DL2/' + night + '/' + version + '/' + cleaning + '/dl2*' # Current format of LST data path
if args.verbose:
print("looking for files here :", folder)
filepath_glob = glob(folder)
filelist = pd.DataFrame(columns=['path'])
# Create a list of files with matching run numbers
newEntry = {}
for filename in filepath_glob:
for run in runs:
if (f"Run{run:>05}.h5" in filename):
newEntry['path'] = filename
filelist.loc[run] = newEntry
return filelist
def manage_submission(args, config, cmd, run, level="3"):
"""
Parameters
----------
config: personal configuration file with analysis settings
cmd : command to be run
run : run number
"""
print("Submission of the jobs to the slurm farm.")
os.makedirs(config['jobmanager'], exist_ok=True)
template = open(os.environ.get('CODE_DIR') + "/SubmitTemplate_dl" + level + ".sh", "r").readlines()
scriptname = config['jobmanager'] + "/Script_dl" + level + "_" + str(run) + ".sh"
logfile = config['jobmanager'] + "/Slurm_dl" + level + "_" + str(run) + ".out"
script = open(scriptname, "w")
for t in template:
script.write(t.replace("jobname", "Job_DL" + level + "_" + f"Run{run:>05}").replace("logfile", logfile))
script.write("\n")
script.write(cmd)
script.close()
os.system("chmod +x " + scriptname)
return scriptname
def get_coordinates(args):
"""
returns the name and the Ra/Dec of the source
"""
if args.source_name is None:
raise ValueError("Please provide the name of the analysed source by using --source_name")
if args.verbose:
print("Search Coordinates for ", args.source_name, " using Astropy and based on the target name.")
try:
c = SkyCoord.from_name(args.source_name)
ra = c.ra.value
dec = c.dec.value
success = True
if args.verbose:
print("Coordinates of ", args.source_name, "found using Astropy: RA ", ra, ", Dec ", dec)
print("Cannot resolve target name", args.source_name, "using Astropy. Switch to the Ra and Dec provided by the user.")
if all(item is not None for item in [args.ra, args.dec]):
if args.ra >= 0 and args.ra < 360 and args.dec >= -90 and args.dec < 90:
ra = args.ra
dec = args.dec
if args.verbose:
print("Coordinates provided by the user: RA ", ra, ", Dec", dec)
else:
print("Please provide RA and Dec values by using --ra and --dec")
exit(0)
if (success is True and args.ra and args.dec):
if args.ra != ra or args.dec != dec:
print(f"WARNING! Astropy coordinates RA {ra}, Dec {dec} are different than the ones provided by the user RA {args.ra}, Dec {args.dec}.")
def print_runs(table, mask, by_date=False):
"""
function to print out the run numbers that survive a certain set of cuts
"""
print(f"{mask.sum()} wobble runs for the selected source")
print(f"Observation time: {table['elapsed_time'][mask].sum()/3600:.2f} hours")
print()
print(np.array2string(np.array(table['runnumber'][mask]), separator=', '))
if by_date:
print()
print()
dates = [datetime.utcfromtimestamp(t - 0.5 * 86400).date() for t in table['time'][mask]]
for i, date in enumerate(np.unique(dates)):
rr = []
for d, run in zip(dates, table['runnumber'][mask]):
if d != date:
continue
rr.append(run)
print(i + 1, ":", date, ":", rr)