add dl2 to dl3 scripts

DL2_to_DL3.py

+import os
+from pathlib import Path
+from utils import (
+    parse_arguments,
+    get_config,
+    manage_submission,
+    get_coordinates,
+)
+
+# Argument parsera
+args = parse_arguments(description="DL2 to DL3 converter",
+                       add_job=True,
+                       add_run=True,
+                       add_dl3=True)
+
+config_file = get_config(args.config)
+
+# manage the target
+ra, dec = get_coordinates(args)
+
+# manage the cuts
+if args.cut_file is not None:
+    cut_file = config_file['cut_folder'] + "/" + args.cut_file
+    print("Using cut file:", cut_file)
+else:
+    print("Standard cuts will be used in the analysis.")
+
+irf = config_file['irf_file']
+
+for night in config_file['dl2_data']['night']:
+    # output directory
+    dl3_out_dir = config_file['base_dir'] + '/DL3/' + args.source_name + '/' + str(night) + '/' + config_file['dl2_data']['version'] + '/' + config_file['dl2_data']['cleaning']
+    if args.outdir is not None:
+        dl3_out_dir = args.outdir
+
+    dl2_dir = config_file['base_dir'] + '/DL2/' + args.source_name + '/' + str(night) + '/' + config_file['dl2_data']['version'] + '/' + config_file['dl2_data']['cleaning']
+    files = Path(dl2_dir).glob('dl2*.h5')
+    
+    for file in files:
+        dl3_cmd = (
+            f'lstchain_create_dl3_file -d {file} '
+            f'-o {dl3_out_dir} '
+            f'--input-irf {irf} '
+            f'--source-name {args.source_name} '
+            f'--source-ra "{ra}deg" '
+            f'--source-dec "{dec}deg" '
+            f'{"--overwrite " if args.globber is not False else ""}'
+        )
+        if args.cut_file is not None:
+            dl3_cmd += f'--config {args.cut_file}'
+        if args.gh_cut:
+            dl3_cmd += f'--fixed-gh-cut {args.gh_cut}'
+        if args.theta_cut:
+            dl3_cmd += f'--fixed-theta-cut {args.theta_cut}'
+
+        if args.verbose:
+            print('\n')
+            print(dl3_cmd)
+        if not(args.dry):
+            if args.submit:
+                # create the script that will be submited and return the name of the script in the variable scriptname
+                scriptname = manage_submission(args, config_file, dl3_cmd, os.path.basename(file))
+                os.system("sbatch " + scriptname)
+            else:
+                print('Interactive conversion of the DL2 to DL3.')
+                os.system(dl3_cmd)

SubmitTemplate_dl3.sh

+#!/bin/sh
+#SBATCH -p short,long           
+#SBATCH -J jobname
+#SBATCH --mem=9g
+#SBATCH -N 1
+#SBATCH --exclusive
+#SBATCH -o logfile
+
+ulimit -l unlimited
+ulimit -s unlimited
+ulimit -a

config_dl2_to_dl3.yaml

+# Directory where job files are written
+jobmanager: ../jobmanager
+
+# Database file name
+db: database.csv
+
+# Path to main data folder of the user
+# Change it accordingly to your working env
+base_dir: /fefs/aswg/workspace/alice.donini/Analysis/data
+
+# Path to personal directory where output data will be saved.
+# Uncomment and modify in case you want to use a non standard path
+#output_folder: ../DL3/Crab
+
+# Path to the folder where cut files are stored
+cut_folder: ../cuts
+
+# Path to IRF file, change it based on your analysis
+irf_file: "/fefs/aswg/data/mc/IRF/20200629_prod5_trans_80/zenith_20deg/south_pointing/20210416_v0.7.3_prod5_trans_80_local_taicut_8_4/off0.4deg/irf_20210416_v073_prod5_trans_80_local_taicut_8_4_gamma_point-like_off04deg.fits.gz"
+
+# Values for automatic selection of DL2 data from personal directories
+dl2_data:
+  night: [20210911, 20210912]           # day(s) of observation (more than one is possible)
+  version: v0.9.1                       # v0.7.3, v0.8.4, v0.9, v0.9.1
+  cleaning: tailcut84
+
+
+# Old settings in case using a database
+# Values for automatic selection of DL2 data
+#dl2_data:
+#  DL2_dir: /fefs/aswg/data/real/DL2     # path to DL2 directory
+#  night: [20210911, 20210912]           # day(s) of observation (more than one is possible)
+#  version: v0.9.1                       # v0.7.3, v0.8.4, v0.9, v0.9.1
+#  cleaning: tailcut84
+
+# LST real data path (don't modify it)
+#data_folder: /fefs/aswg/data/real

create_DL3_index.py

+import os
+from utils import (parse_arguments,
+                   get_config)
+
+# Argument parser
+args = parse_arguments(description="DL3 index maker",
+                       add_job=True,
+                       add_run=True)
+
+config_file = get_config(args.config)
+
+for night in config_file['dl2_data']['night']:
+    # output directory
+    dl3_out_dir = config_file['base_dir'] + '/DL3/' + args.source_name + '/' + str(night) + '/' + config_file['dl2_data']['version'] + '/' + config_file['dl2_data']['cleaning']
+    if args.outdir is not None:
+        dl3_out_dir = args.outdir
+
+    # check if directory exists
+    if not (os.path.isdir(dl3_out_dir)):
+        raise Exception(f'Output directory {dl3_out_dir} doesn\'t exists.')
+
+    cmd_index = "lstchain_create_dl3_index_files --input-dl3-dir " + dl3_out_dir + " --file-pattern dl3*fits.gz --overwrite"
+
+    if args.verbose:
+        print('\n')
+        print(cmd_index)
+    if not(args.dry):
+        os.system(cmd_index)