add dl2 to dl3 scripts

import os

from pathlib import Path

from utils import (

    parse_arguments,

    get_config,

    manage_submission,

    get_coordinates,

)

# Argument parsera

args = parse_arguments(description="DL2 to DL3 converter",

                       add_job=True,

                       add_run=True,

                       add_dl3=True)

config_file = get_config(args.config)

# manage the target

ra, dec = get_coordinates(args)

# manage the cuts

if args.cut_file is not None:

    cut_file = config_file['cut_folder'] + "/" + args.cut_file

    print("Using cut file:", cut_file)

else:

    print("Standard cuts will be used in the analysis.")

irf = config_file['irf_file']

for night in config_file['dl2_data']['night']:

    # output directory

    dl3_out_dir = config_file['base_dir'] + '/DL3/' + args.source_name + '/' + str(night) + '/' + config_file['dl2_data']['version'] + '/' + config_file['dl2_data']['cleaning']

    if args.outdir is not None:

        dl3_out_dir = args.outdir

    dl2_dir = config_file['base_dir'] + '/DL2/' + args.source_name + '/' + str(night) + '/' + config_file['dl2_data']['version'] + '/' + config_file['dl2_data']['cleaning']

    files = Path(dl2_dir).glob('dl2*.h5')

    for file in files:

        dl3_cmd = (

            f'lstchain_create_dl3_file -d {file} '

            f'-o {dl3_out_dir} '

            f'--input-irf {irf} '

            f'--source-name {args.source_name} '

            f'--source-ra "{ra}deg" '

            f'--source-dec "{dec}deg" '

            f'{"--overwrite " if args.globber is not False else ""}'

        )

        if args.cut_file is not None:

            dl3_cmd += f'--config {args.cut_file}'

        if args.gh_cut:

            dl3_cmd += f'--fixed-gh-cut {args.gh_cut}'

        if args.theta_cut:

            dl3_cmd += f'--fixed-theta-cut {args.theta_cut}'

        if args.verbose:

            print('\n')

            print(dl3_cmd)

        if not(args.dry):

            if args.submit:

                # create the script that will be submited and return the name of the script in the variable scriptname

                scriptname = manage_submission(args, config_file, dl3_cmd, os.path.basename(file))

                os.system("sbatch " + scriptname)

            else:

                print('Interactive conversion of the DL2 to DL3.')

                os.system(dl3_cmd)

#!/bin/sh

#SBATCH -p short,long           

#SBATCH -J jobname

#SBATCH --mem=9g

#SBATCH -N 1

#SBATCH --exclusive

#SBATCH -o logfile

ulimit -l unlimited

ulimit -s unlimited

ulimit -a

# Directory where job files are written

jobmanager: ../jobmanager

# Database file name

db: database.csv

# Path to main data folder of the user

# Change it accordingly to your working env

base_dir: /fefs/aswg/workspace/alice.donini/Analysis/data

# Path to personal directory where output data will be saved.

# Uncomment and modify in case you want to use a non standard path

#output_folder: ../DL3/Crab

# Path to the folder where cut files are stored

cut_folder: ../cuts

# Path to IRF file, change it based on your analysis

irf_file: "/fefs/aswg/data/mc/IRF/20200629_prod5_trans_80/zenith_20deg/south_pointing/20210416_v0.7.3_prod5_trans_80_local_taicut_8_4/off0.4deg/irf_20210416_v073_prod5_trans_80_local_taicut_8_4_gamma_point-like_off04deg.fits.gz"

# Values for automatic selection of DL2 data from personal directories

dl2_data:

  night: [20210911, 20210912]           # day(s) of observation (more than one is possible)

  version: v0.9.1                       # v0.7.3, v0.8.4, v0.9, v0.9.1

  cleaning: tailcut84

# Old settings in case using a database

# Values for automatic selection of DL2 data

#dl2_data:

#  DL2_dir: /fefs/aswg/data/real/DL2     # path to DL2 directory

#  night: [20210911, 20210912]           # day(s) of observation (more than one is possible)

#  version: v0.9.1                       # v0.7.3, v0.8.4, v0.9, v0.9.1

#  cleaning: tailcut84

# LST real data path (don't modify it)

#data_folder: /fefs/aswg/data/real

import os

from utils import (parse_arguments,

                   get_config)

# Argument parser

args = parse_arguments(description="DL3 index maker",

                       add_job=True,

                       add_run=True)

config_file = get_config(args.config)

for night in config_file['dl2_data']['night']:

    # output directory

    dl3_out_dir = config_file['base_dir'] + '/DL3/' + args.source_name + '/' + str(night) + '/' + config_file['dl2_data']['version'] + '/' + config_file['dl2_data']['cleaning']

    if args.outdir is not None:

        dl3_out_dir = args.outdir

    # check if directory exists

    if not (os.path.isdir(dl3_out_dir)):

        raise Exception(f'Output directory {dl3_out_dir} doesn\'t exists.')

    cmd_index = "lstchain_create_dl3_index_files --input-dl3-dir " + dl3_out_dir + " --file-pattern dl3*fits.gz --overwrite"

    if args.verbose:

        print('\n')

        print(cmd_index)

    if not(args.dry):

        os.system(cmd_index)