add utils file

utils.py

+import argparse
+import os
+import pandas as pd
+import numpy as np
+import yaml
+from operator import attrgetter
+from astropy.coordinates import SkyCoord
+from glob import glob
+from argparse import HelpFormatter
+
+
+class SortingHelpFormatter(HelpFormatter):
+    def add_arguments(self, actions):
+        actions = sorted(actions, key=attrgetter('option_strings'))
+        super(SortingHelpFormatter, self).add_arguments(actions)
+
+
+def parse_arguments(description, add_run=False, add_irf=False, add_job=False, add_dl3=False):
+    parser = argparse.ArgumentParser(description=description, formatter_class=SortingHelpFormatter)
+
+    parser.add_argument('--prod', '-p',
+                        dest='prod', required=False, type=str, default='v0.9.4',
+                        help='Prod to use (default: %(default)s)')
+    parser.add_argument('--outdir', '-o',
+                        dest='outdir', required=False, type=str, default=None,
+                        help='Directory to store the output')
+    parser.add_argument('--config', '-c',
+                        type=str, default=None, dest='config',
+                        help='Specify a personal config file for the analysis')
+    parser.add_argument('--config-analysis',
+                        type=str, default=None, dest='config_analysis',
+                        help='Specify a config file which describes analysis profile to use')
+    parser.add_argument('--verbose', '-v',
+                        action='store_true', dest='verbose',
+                        help='Increase output verbosity')
+
+    if add_run:
+        parser.add_argument('--source_name', '-n', required=True,
+                            default=None, dest='source_name', type=str,
+                            help='Name of the source')
+        parser.add_argument('--tcuname',
+                            default=None, dest='tcuname', type=str,
+                            help='Apply run selection based on TCU source name')
+        parser.add_argument('--runlist', '-rl',
+                            default=None, dest='runlist', type=str,
+                            help='File with a list of run and the associated night to be analysed')
+        parser.add_argument('--distance', '-dis',
+                            type=float, dest='distance', default=-1,
+                            help='Max distance in degrees between the target position and the run pointing \
+                                  position for the run selection, negative value means no selection using \
+                                  this parameter (default: %(default)s).')
+        parser.add_argument('--ra',
+                            type=float, dest='ra', default=-1,
+                            help='RA coordinate of the target. To add if you want to use custom position')
+        parser.add_argument('--dec',
+                            type=float, dest='dec', default=-91,
+                            help='Dec coordinate of the target. To add if you want to use custom position')
+
+    if add_job:
+        parser.add_argument('--submit',
+                            default=False, dest='submit', action='store_true', required=False,
+                            help='Submit the cmd to slurm on site')
+        parser.add_argument('--dry',
+                            default=False, required=False, action='store_true', dest='dry',
+                            help='Make a dry run, no true submission')
+        parser.add_argument('--globber', '-g',
+                            dest='globber', action='store_true', required=False, default=False,
+                            help='If True, overwrites existing output file without asking')
+
+    if add_irf:
+        parser.add_argument('--gh_cut',
+                            required=False, type=float, dest='gh_cut',
+                            help='Fixed selection cut for gh_score (gammaness)')
+        parser.add_argument('--theta_cut',
+                            required=False, type=float, dest='theta_cut',
+                            help='Fixed selection cut for theta')
+        parser.add_argument('--obs_time',
+                            required=False, type=float, dest='obs_time',
+                            help='Observation time for IRF in hours')
+
+    if add_dl3:
+        parser.add_argument('--cut_file', '-cut',
+                            default=None, dest='cut_file', type=str,
+                            help='Cut file')
+        parser.add_argument('--gh_cut',
+                            required=False, type=float, dest='gh_cut',
+                            help='Fixed selection cut for gh_score (gammaness)')
+        parser.add_argument('--theta_cut',
+                            required=False, type=float, dest='theta_cut',
+                            help='Fixed selection cut for theta')
+    args = parser.parse_args()
+    return args
+
+
+def get_db(database_filename):
+    """
+    Parameters
+    ----------
+    database_filename : name of database file
+    
+    ---------
+    Returns the DB
+    """
+
+    db_file = os.environ.get('CONFIG_FOLDER') + '/' + database_filename
+    database = pd.read_csv(db_file, index_col=0, parse_dates=True)
+    return database
+
+
+def get_config(config_filename):
+    """
+    Parameters
+    ----------
+    config_filename : name of the configuration file
+    
+    ---------
+    Returns the configuration file
+    """
+
+    config_file = os.environ.get('CONFIG_FOLDER') + '/' + config_filename
+    with open(config_file) as f:
+        config_analysis = yaml.load(f, Loader=yaml.FullLoader)
+    return config_analysis
+
+
+def get_runs_database(args, database):
+
+    # make the run list into a np array
+    databaseRuns = np.array(database.index)
+
+    # Apply selection of data if argument is specified. Based on tcuname, run or coordinates
+    selection = database
+
+    #if args.tcuname[0] == 'all' and args.night[0] == 'all' and args.runlist[0] == 'none':
+    #    raise RuntimeError("Cannont make a run selection. Either tcuname, night or runlist or distance is needed")
+
+    if args.tcuname is not None:
+        selection = database.loc[database['Target']].isin([args.tcuname])
+        if args.verbose:
+            print("Selection of runs based on the TCU name", args.tcuname, ". Only run with the name in the TCU are kept")
+            print(selection.index)
+
+    # if args.night[0] != 'all':
+    #     selection = database.loc[database['day'].isin(args.night)]
+    #     if args.verbose:
+    #         print("selection of night", args.night[0])
+    #         print(selection.index)
+    
+    if args.distance > 0:
+        if args.ra == -1 or args.dec == -91:
+            raise RuntimeError("Cannont make a run selection. Ra and Dec value of the source are not given.")
+        selection = database.loc[((database['RA_Obs'] - args.ra)**2 + (database['Dec_Obs'] - args.dec)**2) < args.distance**2]
+        if args.verbose:
+            print("Selection based on angular distance ", args.distance, "°")
+            print(selection.index)
+
+    databaseRuns = np.array(selection.index)
+    if args.runlist is not None:
+        rl = np.loadtxt(args.runlist, unpack=True, dtype=int)
+        #databaseRuns = np.array([a for a in rl if a in databaseRuns])
+        databaseRuns = np.array([a for a in rl])
+
+    if args.verbose:
+        print("Final run selection", databaseRuns)
+    return databaseRuns
+
+
+def create_DL1_list(args, config, runs, night):
+    """
+    create list with all the DL1 run path to analyze
+    
+    Parameters
+        ----------
+        runs: List of run numbers
+        night: night to analyze
+    """
+
+    version = config['dl1_data']['version']
+    cleaning = config['dl1_data']['cleaning']
+    folder = config['data_folder'] + '/DL1/' + night + '/' + version + '/' + cleaning + '/dl1*'  # Current format of LST data path
+    if args.verbose:
+        print("Looking for files here :", folder)
+
+    filepath_glob = glob(folder)
+
+    # initializa data frame
+    filelist = pd.DataFrame(columns=['path', 'night'])
+
+    # Create a list of files with matching run numbers
+    newEntry = {}
+    for filename in filepath_glob:
+        for run in runs:
+            if (f"Run{run:>05}.h5" in filename):
+                newEntry['path'] = filename
+                newEntry['night'] = night
+                filelist.loc[run] = newEntry['path'], newEntry['night']
+    return filelist
+
+
+def create_DL2_list(args, config, runs, night):
+    """
+        create list with all the DL2 run path to analyze
+    
+    Parameters
+    ----------
+    runs: List of run numbers
+    """
+    
+    version = config['dl2_data']['version']
+    cleaning = config['dl2_data']['cleaning']
+    folder = config['data_folder'] + '/DL2/' + night + '/' + version + '/' + cleaning + '/dl2*'  # Current format of LST data path
+    if args.verbose:
+        print("looking for files here :", folder)
+
+    filepath_glob = glob(folder)
+
+    filelist = pd.DataFrame(columns=['path'])
+
+    # Create a list of files with matching run numbers
+    newEntry = {}
+    for filename in filepath_glob:
+        for run in runs:
+            if (f"Run{run:>05}.h5" in filename):
+                newEntry['path'] = filename
+                filelist.loc[run] = newEntry
+    return filelist
+
+
+def manage_submission(args, config, cmd, run, level="3"):
+    """
+    Parameters
+    ----------
+    config: personal configuration file with analysis settings
+    cmd : command to be run
+    run : run number
+    """
+
+    print("Submission of the jobs to the slurm farm.")
+
+    os.makedirs(config['jobmanager'], exist_ok=True)
+    template = open(os.environ.get('CODE_DIR') + "/SubmitTemplate_dl" + level + ".sh", "r").readlines()
+
+    scriptname = config['jobmanager'] + "/Script_dl" + level + "_" + str(run) + ".sh"
+    logfile = config['jobmanager'] + "/Slurm_dl" + level + "_" + str(run) + ".out"
+    script = open(scriptname, "w")
+    for t in template:
+        script.write(t.replace("jobname", "Job_DL" + level + "_" + f"Run{run:>05}").replace("logfile", logfile))
+
+    script.write("\n")
+    script.write(cmd)
+    script.close()
+    os.system("chmod +x " + scriptname)
+    return scriptname
+
+
+def get_coordinates(args):
+    """
+    returns the name and the Ra/Dec of the source
+    """
+
+    if args.source_name is None:
+        raise ValueError("Please provide the name of the analysed source by using --source_name")
+
+    if args.verbose:
+        print("Search Coordinates for ", args.source_name, " using Astropy and based on the target name.")
+    try:
+        c = SkyCoord.from_name(args.source_name)
+        ra = c.ra.value
+        dec = c.dec.value
+    except:
+        print("Cannot resolve target name", args.source_name, "using Astropy. Switch to the Ra and Dec provided.")
+        if args.ra >= 0 and args.ra < 360 and args.dec >= -90 and args.dec < 90:
+            ra = args.ra
+            dec = args.dec
+            print("Using user provided RA and Dec.")
+        else:
+            print("Please provide RA and Dec values by using --ra and --dec")
+            exit(0)
+    return ra, dec