Skip to content
Commits on Source (8)
......@@ -14,8 +14,9 @@ from utils import (
# Argument parser
args = parse_arguments(description="DL1 to DL2 converter",
add_run=True,
add_dl2=True,
add_job=True,
add_common=True,
)
config_file = get_config(args.config)
......@@ -28,9 +29,17 @@ final_run_list = pd.DataFrame(columns=['path'])
# if txt file with runs and nights is given, use that instead search in the database
if args.runlist is not None:
rl = np.genfromtxt(open(args.runlist, 'r'), usecols=(0,1), dtype=int)
runs = rl[:,0]
nights = rl[:,1]
runlist = os.environ.get('CONFIG_FOLDER') + '/' + args.runlist
rl = np.genfromtxt(open(runlist, 'r'), usecols=(0, 1), dtype=int)
n_element = np.count_nonzero(rl)
if n_element == 2:
runs = np.array([rl[0]])
nights = np.array([rl[1]])
else:
runs = rl[:, 0]
nights = rl[:, 1]
# runs = rl[:,0]
# nights = rl[:,1]
# nights = []
# for run in rl:
# night = database_file.loc[run, 'day']
......@@ -48,6 +57,7 @@ else:
final_run_list = pd.concat([final_run_list, run_list])
if args.verbose:
print('')
print("Final run list to be analysed:")
print(final_run_list)
......@@ -57,6 +67,12 @@ out_dir = config_file['base_dir'] + '/DL2/' + args.source_name
# path to model directory
models = config_file['path_models']
# custom lstchain config file
if 'lstchain_config' in config_file:
config_lstchain = config_file['lstchain_config']
else:
config_lstchain = None
# check if output directory exists, if not, create it
if not (os.path.isdir(out_dir)):
try:
......@@ -82,7 +98,7 @@ if args.runlist is not None:
--input-file {dl1_file} \
--output-dir {dl2_output_path} \
--path-models {models}' \
+ ['', f' --config {args.config_analysis}'][args.config_analysis is not None]
+ ['', f' --config {config_lstchain}'][config_lstchain is not None]
# Send interactive, sbatch or dry run
if args.verbose:
......@@ -112,7 +128,7 @@ else:
f'--output-dir {dl2_output_path} '
f'--path-models {models} '
) \
+ ['', f' --config {args.config_analysis}'][args.config_analysis is not None]
+ ['', f' --config {config_lstchain}'][config_lstchain is not None]
# Send interactive, sbatch or dry run
if args.verbose:
......
......@@ -10,8 +10,8 @@ from utils import (
# Argument parsera
args = parse_arguments(description="DL2 to DL3 converter",
add_job=True,
add_run=True,
add_dl3=True)
add_dl3=True,
add_common=True)
config_file = get_config(args.config)
......
import tables
run = '8623'
data_dir = '/fefs/aswg/data/real/DL1' # path to DLx directory
night = '20220603' # day of observation
version = 'v0.9' # v0.7.3, v0.8.4, v0.9, v0.9.1
cleaning = 'tailcut84'
path = data_dir + '/' + night + '/' + version + '/' + cleaning
filename = 'dl1_LST-1.Run0' + run + '.h5'
file = path + '/' + filename
print("file: ", file)
with tables.open_file(file) as file:
print(file.root._v_attrs["LSTCHAIN_VERSION"])
......@@ -13,17 +13,18 @@ base_dir: /fefs/aswg/workspace/alice.donini/Analysis/data
# Path to personal directory where output data will be saved.
# Uncomment and modify in case you want to use a non standard path
#output_folder: ../DL2/Crab
# Directory where config files are stored
#config_folder: ./
# Otherwise files will be saved to {base_dir}/DL2/{source_name/{night}/{version}/{cleaning}
#dl2_output_folder: ../DL2/Crab
# Path to trained RF files
path_models: ../models
# Uncomment and modify in case you want to specify a custom configuration file for the lstchain script
# lstchain_config: lstchain_84.json
# Values for automatic selection of DL1 data
dl1_data:
DL1_dir: /fefs/aswg/data/real/DL1 # path to DL1 directory
night: [20210911, 20210912] # day(s) of observation (more than one is possible)
version: v0.9.1 # v0.7.3, v0.8.4, v0.9, v0.9.1
version: v0.9 # v0.7.3, v0.8.4, v0.9
cleaning: tailcut84
# uncomment the line below and specify the nights only if database search is used and not a custom runlist file
#night: [20210911, 20210912] # day(s) of observation (more than one is possible)
......@@ -4,8 +4,8 @@ from utils import (parse_arguments,
# Argument parser
args = parse_arguments(description="DL3 index maker",
add_job=True,
add_run=True)
add_common=True,
add_job=False)
config_file = get_config(args.config)
......
......@@ -70,7 +70,7 @@ def main():
parser.add_argument(
'--version', type=str, required=False, dest='version',
default='v0.9.2', help='lstchain version (default: %(default)s)'
default='v0.9', help='lstchain version (default: %(default)s)'
)
parser.add_argument(
......
import glob
import os
import os.path
import tables
import numpy as np
import astropy.units as u
from astropy.table import vstack
from astropy.coordinates import SkyCoord
from datetime import datetime, timedelta
from ctapipe.io import read_table
from utils import (
parse_arguments,
get_coordinates,
print_runs,
)
# Argument parser
args = parse_arguments(description="Runlist creation",
create_runlist=True,
)
user = False
source = args.source_name
if args.verbose:
print(f'Selected source: {source}')
# overwrite search through astropy if user specifies coordinates
if args.ra and args.dec:
ra = args.ra
dec = args.dec
user = True
else:
ra, dec = get_coordinates(args)
source_coordinates = SkyCoord(ra=ra * u.deg, dec=dec * u.deg)
if user is True and args.verbose:
print(f'Coordinates in deg of selected source given by the user: {source_coordinates}')
# Load all the available night-wise DL1_datacheck_YYYYMMDD.h5 files (or at least those of the nights which contain the data you intend to analyze)
# The files are available in the IT cluster under the directories: /fefs/aswg/data/real/OSA/DL1DataCheck_LongTerm/v0.9/YYYYMMDD/
datacheck_files = []
if args.night:
add_date = []
for night in args.night:
sub_day = datetime.strptime(night, '%Y%m%d').date() - timedelta(days=1)
add_date.append(sub_day.strftime('%Y%m%d'))
for date in add_date:
if date not in args.night:
args.night.append(date)
args.night.sort(key=lambda x: datetime.strptime(x, '%Y%m%d'))
if args.verbose:
print(f"Nights selected: {args.night}.")
for night in args.night:
datacheck_file = glob.glob(f"/fefs/aswg/data/real/OSA/DL1DataCheck_LongTerm/v0.9/{night}/DL1_datacheck_20*.h5")
datacheck_files.append(datacheck_file[0])
else:
datacheck_files = glob.glob("/fefs/aswg/data/real/OSA/DL1DataCheck_LongTerm/v0.9/20*/DL1_datacheck_20*.h5")
datacheck_files.sort()
if args.verbose:
print(f'A total of {len(datacheck_files)} files will be read')
dcheck_runsummary = []
for file in datacheck_files:
# print(file)
# Check that the file contains the necessary info
with tables.open_file(file) as a:
if "pedestals" not in a.root:
if args.verbose:
print(f"file {file} does not contain the interleaved pedestals table... Skipping!!")
continue
if "flatfield" not in a.root:
if args.verbose:
print(f"file {file} does not contain the interleaved flatfield table... Skipping!!")
continue
table = read_table(file, "/runsummary/table")
# Set to 0 the number of flatfield events in nan (means none was found)
table['num_flatfield'] = np.where(np.isnan(table['num_flatfield']), 0, table['num_flatfield'])
dcheck_runsummary.append(table)
# Ignore metadata conflicts below. i.e. metadata_conflicts='silent'.
# It is just that columns which contain some nan are float64, while those which do not are float32
dcheck_runsummary = vstack(dcheck_runsummary, metadata_conflicts='silent')
# Exclude runs with issues in interleaved pedestals
ped_ok_mask = np.isfinite(dcheck_runsummary['num_pedestals'])
print('Removed runs:', np.array(dcheck_runsummary['runnumber'][~ped_ok_mask]))
# Find the runs with pointing close to the source of interest
telescope_pointing = SkyCoord(ra=dcheck_runsummary['mean_ra'] * u.deg, dec=dcheck_runsummary['mean_dec'] * u.deg)
angular_distance = source_coordinates.separation(telescope_pointing)
# Select wobble pointings at ~0.4 deg from the source:
source_mask = ((angular_distance > 0.35 * u.deg) & (angular_distance < 0.45 * u.deg))
print_runs(dcheck_runsummary, source_mask, by_date=True)
dates = [datetime.utcfromtimestamp(t - 0.5 * 86400).strftime('%Y%m%d') for t in dcheck_runsummary['time'][source_mask]]
dcheck_runsummary_masked = dcheck_runsummary['runnumber', 'time'][source_mask]
dcheck_runsummary_masked['date'] = dates
del dcheck_runsummary_masked['time']
if args.verbose:
print(dcheck_runsummary_masked)
# write the txt file
dcheck_runsummary_masked.write(os.environ.get('CONFIG_FOLDER') + f'/{source}_runlist.txt', format='pandas.csv', header=False, sep=' ')
if args.verbose:
if os.path.exists(os.environ.get('CONFIG_FOLDER') + f'/{source}_runlist.txt'):
print('')
print(f'Runlist file {source}_runlist.txt created in {os.environ.get("CONFIG_FOLDER")}')
else:
print('ERROR: Not able to create runlist file.')
......@@ -2,7 +2,7 @@
A "How to" Guide for LST analysis at IT
===============================================================================================
This document aims to describe how to perform an analysis at the IT cluster in La Palma using the scripts hosted on the `GitLab repository <https://www.ict.inaf.it/gitlab/alice.donini/lst-analysis>`_ .
This document aims to describe how to perform an analysis at the IT cluster in La Palma using the scripts hosted on the `GitLab repository <https://gitlab.com/davide.miceli1993/lst_scripts>`_ .
.. toctree::
......@@ -18,4 +18,4 @@ This document aims to describe how to perform an analysis at the IT cluster in L
.. * :ref:`genindex`
.. * :ref:`modindex`
.. * :ref:`search`
.. * :ref:`search`
\ No newline at end of file
.. _dl1:
DL1 file generation
===================
......@@ -4,33 +4,29 @@ DL2 files generation
====================
The script `DL1_to_DL2.py` allows to analyze DL1 data and produce DL2 files.
Mandatory arguments are the name of the source (option ``-n``) and a configuration file `config_dl1_to_dl2.yaml` (option ``--config``). The script will search for the configuration file in the ``$CONFIG`` folder specified in the initial settings.
Usage:
.. code-block::
usage: DL1_to_DL2.py [-h] [--prod PROD] [--outdir OUTDIR] [--config CONFIG] [--config-analysis CONFIG_ANALYSIS]
[--verbose] [--source_name SOURCE_NAME] [--tcuname TCUNAME] [--runlist RUNLIST]
[--distance DISTANCE] [--ra RA] [--dec DEC] [--submit] [--dry] [--globber]
usage: DL1_to_DL2.py [-h] [--verbose] --source_name SOURCE_NAME --config CONFIG [--dry] [--outdir OUTDIR] [--tcuname TCUNAME] [--runlist RUNLIST] [--distance DISTANCE] [--ra RA] [--dec DEC]
[--submit] [--globber]
DL1 to DL2 converter
optional arguments:
--config CONFIG, -c CONFIG
Specify a personal config file for the analysis
--config-analysis CONFIG_ANALYSIS
Specify a config file which described analysis profile to use
--dec DEC Dec coordinate of the target. To add if you want to use custom position
--distance DISTANCE, -dis DISTANCE
Max distance in degrees between the target position and the run pointing position for the
run selection, negative value means no selection using this parameter (default: -1).
Max distance in degrees between the target position and the run pointing position for the run selection. Negative value means no selection using this parameter (default: -1).
--dry Make a dry run, no true submission
--globber, -g If True, overwrites existing output file without asking
--globber, -g If True, overwrites existing output file without asking (default: False).
--outdir OUTDIR, -o OUTDIR
Directory to store the output
--prod PROD, -p PROD Prod to use (default: v0.8.4)
--ra RA RA coordinate of the target. To add if you want to use custom position
--runlist RUNLIST, -rl RUNLIST
File with a list of run and the associated night to be analysed
File with a list of run and the associated night to be analysed
--source_name SOURCE_NAME, -n SOURCE_NAME
Name of the source
--submit Submit the cmd to slurm on site
......@@ -38,11 +34,11 @@ Usage:
--verbose, -v Increase output verbosity
-h, --help show this help message and exit
It makes use of a configuration file `config_dl1_to_dl2.yaml` (option ``--config``):
Preview of the configuration file:
.. code-block:: yaml
# Directory where job file are written
# Directory where job files are written
jobmanager: ../jobmanager
# Database file name
......@@ -53,42 +49,45 @@ It makes use of a configuration file `config_dl1_to_dl2.yaml` (option ``--config
# path to main data folder of the user
# change it accordingly to your working env
base_dir: /fefs/aswg/alice.donini/Analysis/data
base_dir: /fefs/aswg/workspace/alice.donini/Analysis/data
# Path to personal directory where output data will be saved.
# Uncomment and modify in case you want to use a non standard path
#output_folder: ../DL2/Crab
# Directory where config files are stored
#config_folder: ./
# Otherwise files will be saved to {base_dir}/DL2/{source_name}/{night}/{version}/{cleaning}
#dl2_output_folder: ../DL2/Crab
# Path to trained RF files
path_models: ../models
# Uncomment and modify in case you want to specify a custom configuration file for the lstchain script
# lstchain_config: ../lstchain_84.json
# Values for automatic selection of DL1 data
dl1_data:
DL1_dir: /fefs/aswg/data/real/DL1 # path to DL1 directory
night: [20210911, 20210912] # day(s) of observation (more than one is possible)
version: v0.9.1 # v0.7.3, v0.8.4, v0.9, v0.9.1
version: v0.9 # v0.7.3, v0.8.4, v0.9
cleaning: tailcut84
# uncomment the line below and specify the nights only if database search is used and not a custom runlist file
#night: [20210911, 20210912] # day(s) of observation (more than one is possible)
Edit the configuration file: change the paths based on your working directory and modify the DL1 data information used to search for the files at the IT.
The script uses the created database (:ref:`db_generation`) to find the runs to analyze, based on the nights specified in the configuration file `config_dl1_to_dl2.yaml`.
An extra selection can be done in coordinates (``--distance``, ``--ra`` and ``--dec`` are mandatory arguments) or by the name of the source as saved in TCU (argument ``--tcuname``).
There are two ways to select the data to analyze: through a search in the database (outdated for now) or through a run list given by the user.
If you want to use the created database (:ref:`db_generation`) to find the runs to analyze, then fill in the information about the wanted nights in the configuration file `config_dl1_to_dl2.yaml`.
An extra selection can be done in coordinates (in case ``--distance``, ``--ra`` and ``--dec`` are mandatory arguments) or by the name of the source as saved in TCU (argument ``--tcuname``).
If none of these selection methods is given, then all the runs available in the dates specified in the configuration file are considered for the analysis.
The search for DL1 files can be also done by giving a file list of runs and nights to be analyzed (option ``--runlist``).
No database file is needed in this case.
.. warning::
The search of runs through the database has an issue on the dates at the moment. The database is generated from the drive log, so all the runs taken after the midnight are saved under the following day. This doesn't happen at the IT, where the runs are stored under the day of the starting night. So for some runs the search could fail, even if they are there.
The search of runs through the database has a "feature". The database is generated from the drive log, so all the runs taken after the midnight are saved under the following day. This doesn't happen at the IT, where the runs are stored under the day of the starting night. So for some runs the search could fail, even if they are there.
Thus if you use the database search always add in the config file also the date of the night after, so that you are sure all the runs take after the midnight are considered too.
Thus if you use the database search always add in the configuration file also the date of the night after, so that you are sure all the runs take after the midnight are considered too.
Example of a file run list:
The search for DL1 files can be also done by giving a file list of runs and nights to be analyzed (option ``--runlist``).
No database file is needed in this case.
The runlist can be either manually created or produced using the `create_run_list.py` script (:ref:`runlist_generation`).
Example of a runlist file:
.. code-block::
......@@ -96,9 +95,9 @@ Example of a file run list:
2911 20201117
3089 20201206
The argument ``--dry`` allows to perform a dry run.
No jobs are submitted and only the verbose output is printed.
Usefull to have a check of which runs are selected and the goodness of the sent command.
The argument ``--dry`` allows to perform a dry run. No jobs are submitted and only the verbose output is printed.
This option is usefull to have a check of which runs are selected and the goodness of the sent command.
Some examples of how to run the script:
......
......@@ -17,6 +17,7 @@ There are two different scripts to generate the DL3 file:
In the case the run of the job fails due to an astropy error about too old values of the IERS data:
.. code-block::
ValueError: interpolating from IERS_Auto using predictive values that are more than 30.0 days old.
run before the script `IERS_download.py` to download offline the data and update the astropy tables.
......@@ -28,37 +29,25 @@ Usage:
.. code-block::
usage: DL2_to_DL3.py [-h] [--prod PROD] [--outdir OUTDIR] [--config CONFIG] [--config-analysis CONFIG_ANALYSIS]
[--verbose] [--source_name SOURCE_NAME] [--tcuname TCUNAME] [--runlist RUNLIST]
[--distance DISTANCE] [--ra RA] [--dec DEC] [--submit] [--dry] [--globber] [--cut_file CUT]
[--gh_cut GH_CUT] [--theta_cut THETA_CUT]
usage: DL2_to_DL3.py [-h] [--verbose] [--outdir OUTDIR] --config CONFIG --source_name SOURCE_NAME [--ra RA] [--dec DEC] [--cut_file CUT_FILE] [--gh_cut GH_CUT] [--theta_cut THETA_CUT] [--submit] [--dry] [--globber]
DL2 to DL3 converter
optional arguments:
--config CONFIG, -c CONFIG
Specify a personal config file for the analysis
--config-analysis CONFIG_ANALYSIS
Specify a config file which describes analysis profile to use
--cut_file CUT, -cut CUT
--cut_file CUT_FILE, -cut CUT_FILE
Cut file
--dec DEC Dec coordinate of the target. To add if you want to use custom position
--distance DISTANCE, -dis DISTANCE
Max distance in degrees between the target position and the run pointing position for the run
selection, negative value means no selection using this parameter (default: -1).
--dry Make a dry run, no true submission
--gh_cut GH_CUT Fixed selection cut for gh_score (gammaness)
--globber, -g If True, overwrites existing output file without asking
--globber, -g If True, overwrites existing output file without asking (default: False).
--outdir OUTDIR, -o OUTDIR
Directory to store the output
--prod PROD, -p PROD Prod to use (default: v0.9.4)
--ra RA RA coordinate of the target. To add if you want to use custom position
--runlist RUNLIST, -rl RUNLIST
File with a list of run and the associated night to be analysed
--source_name SOURCE_NAME, -n SOURCE_NAME
Name of the source
--submit Submit the cmd to slurm on site
--tcuname TCUNAME Apply run selection based on TCU source name
--theta_cut THETA_CUT
Fixed selection cut for theta
--verbose, -v Increase output verbosity
......
......@@ -22,7 +22,7 @@ Analysis steps
There are many data levels and different scripts are used to call the corresponding ``lstchain`` one. The steps in order are:
* Database Generation (:ref:`db_generation`)
* Database Generation (:ref:`db_generation`) or Runlist generation (:ref: `runlist_generation`)
* DL1 generation (:ref:`dl1`)
* DL2 generation (:ref:`dl2`)
* DL3 generation (:ref:`dl3`)
......@@ -64,13 +64,13 @@ Multiple nights can be specified.
Create a directory structure
optional arguments:
options:
-h, --help show this help message and exit
--main_dir MAIN_DIR Path to parent folder
--source SOURCE Source name
--night NIGHT [NIGHT ...]
Night date
--version VERSION lstchain version (default: v0.9.2)
--version VERSION lstchain version (default: v0.9)
--cleaning CLEANING Cleaning type (default: tailcut84)
Example of use:
......@@ -88,12 +88,13 @@ Output:
Directory /fefs/aswg/workspace/alice.donini/Analysis/data/DL3/Crab/20220304/v0.9.2/tailcut84 created
Directory structure for analysis on Crab was created.
Database generation
Database/Runlist generation
~~~~~~~~~~~~~~~~~~~~
Unless you specify a file with a list of runs and nights, there is the need of a database, through which the run selection is made.
Run selection can be done through two methods:
For the generation of the database file refer to :ref:`db_generation`.
* generation of a file with a list of runs and corresponding nights that is given as input to the analysis scripts, refer to :ref: `runlist_generation`.
* creation of a database through which the run selection is made, refer to :ref:`db_generation`.
DL1 generation
~~~~~~~~~~~~~~~~~~~~~
......
......@@ -10,32 +10,16 @@ Usage:
.. code-block:: bash
usage: create_DL3_index.py [-h] [--prod PROD] [--outdir OUTDIR] [--config CONFIG]
[--config-analysis CONFIG_ANALYSIS] [--verbose] [--source_name SOURCE_NAME]
[--tcuname TCUNAME] [--runlist RUNLIST [RUNLIST ...]] [--distance DISTANCE]
[--ra RA] [--dec DEC]
usage: create_DL3_index.py [-h] [--verbose] --source_name SOURCE_NAME --config CONFIG [--dry]
DL3 index maker
optional arguments:
--config CONFIG, -c CONFIG
Specify a personal config file for the analysis
--config-analysis CONFIG_ANALYSIS
Specify a config file which described analysis profile to use
--dec DEC Dec coordinate of the target. To add if you want to use custom position
--distance DISTANCE, -dis DISTANCE
Max distance in degrees between the target position and the run pointing position
for the run selection, negative value means no selection using this parameter
(default: -1).
--outdir OUTDIR, -o OUTDIR
Directory to store the output
--prod PROD, -p PROD Prod to use (default: v0.8.4)
--ra RA RA coordinate of the target. To add if you want to use custom position
--runlist RUNLIST [RUNLIST ...], -rl RUNLIST [RUNLIST ...]
File with run list to be analysed
--dry Make a dry run, no true submission
--source_name SOURCE_NAME, -n SOURCE_NAME
Name of the source
--tcuname TCUNAME Apply run selection based on TCU source name
--verbose, -v Increase output verbosity
-h, --help show this help message and exit
......
.. _irf_generation:
IRF generation
==============
The script `create_irf.py` allows to create IRFs FITS file from given MC DL2 files and selection cuts. It makes use of the configuration file `config_IRF.yaml`:
.. code-block:: yaml
# path to folder where output data are saved
output_data_folder: ./
# values for automatic mc data selection
DL2_dir: /fefs/aswg/data/mc/DL2/20200629_prod5_trans_80 # path to DL2 directory
version: 20210416_v0.7.3_prod5_trans_80_local_taicut_8_4 # production version, other possibility:
# 20210923_v0.7.5_prod5_trans_80_dynamic_cleaning
zenith: 20 # 20 or 40
direction: south # pointing direction, e.g. south, north
gammatype: point # type of gammas, e.g. point, diffuse
Edit the configuration file: change the path of the output folder based on your working directory and modify the information used to select automatically the MC files.
The script:
.. code-block:: bash
usage: create_irf_files.py [-h] [--prod PROD] [--outdir OUTDIR] [--config CONFIG]
[--config-analysis CONFIG_ANALYSIS] [--verbose] [--submit] [--dry] [--globber]
[--gh_cut GH_CUT] [--theta_cut THETA_CUT] [--obs_time OBS_TIME]
Create IRF from MC files
optional arguments:
--config CONFIG, -c CONFIG
Specify a personal config file for the analysis
--config-analysis CONFIG_ANALYSIS
Specify a config file which described analysis profile to use
--dry Make a dry run, no true submission
--gh_cut GH_CUT Fixed selection cut for gh_score (gammaness)
--globber, -g If True, overwrites existing output file without asking
--obs_time OBS_TIME Observation time for IRF in hours
--outdir OUTDIR, -o OUTDIR
Directory to store the output
--prod PROD, -p PROD Prod to use (default: v0.8.4)
--submit Submit the cmd to slurm on site
--theta_cut THETA_CUT
Fixed selection cut for theta
--verbose, -v Increase output verbosity
-h, --help show this help message and exit
If no analysis configuration file is given, default selection parameters are used (gh=0.6, theta=0.2, time=50h).
For a custom production copy and append the relevant parameters into a custom configuration file, that can be passed with the option ``--config-analysis``. You can take as example the file `irf_tool_config.json`, that can be found in lstchain in `docs/examples/`.
There is the possibility to pass the selection cuts and overwrite the default values directly from the command-line: ``--gh_cut``, ``--theta_cut`` and ``--obs_time``.
These optional arguments can not be used if a custom configuration file is provided.
There is no implementation of job submission (option ``--submit`` is not available).
Examples:
.. code-block:: bash
python create_irf.py -c config_IRF.yaml --config-analysis ../config/irf_tool_config.json -v -g
python create_irf.py -c config_IRF.yaml --gh_cut 0.2 -v -g
.. _runlist_generation:
Runlist generation
===================
The script `create_run_list.py` reads in the DL1 data check information from night-wise hdf5 files, uses them to select runs for analysis and provides a file with a list of the selected runs and their corrisponding night.
If no night is specified trough the option ``--night``, then the script reads in all the DL1 data check files available and this could require some time.
The default folder in which the DL1 data check files are searched is `/fefs/aswg/data/real/OSA/DL1DataCheck_LongTerm/v0.9/20*/`.
The produced runlist file is saved in the ``$CONFIG`` folder specified in the initial settings and can be then used as input to the analysis scripts.
For the selection of the runs:
* Selects runs in which the telescope was pointing towards the source of interest specified.
* Excludes runs in which interleaved pedestals or flatfield events were missing.
.. code-block::
usage: create_run_list.py [-h] [--verbose] --source_name SOURCE_NAME [--ra RA] [--dec DEC] [--night NIGHT [NIGHT ...]]
Runlist creation
optional arguments:
--dec DEC Dec coordinate of the target. To add if you want to use custom position
--night NIGHT [NIGHT ...]
Night of the observation in the format YYYYMMDD. More nights can be specified
--ra RA RA coordinate of the target. To add if you want to use custom position
--source_name SOURCE_NAME, -n SOURCE_NAME
Name of the source
--verbose, -v Increase output verbosity
-h, --help show this help message and exit
In the script it is assumed that the source coordinates are the same as the telescope pointing. So the source name argument ``-n`` is mandatory and is used to find through Astropy the source coordinates, needed from the script to select the runs.
If source coordinates and telescope pointing differ, then specify custom coordinates with arguments ``--ra`` and ``--dec``. These values will overwrite the values obtained through Astropy.
Examples of how to run the script:
.. code-block::
python create_run_list.py -n GRB220527A -v
python create_run_list.py -n GRB220527A --ra 323.56990 --dec -14.90 -v
python create_run_list.py -n GRB210704A --ra 163.73 --dec 58.86 --night 20210705 20210704 20210706 -v
......@@ -7,6 +7,7 @@ from operator import attrgetter
from astropy.coordinates import SkyCoord
from glob import glob
from argparse import HelpFormatter
from datetime import datetime
class SortingHelpFormatter(HelpFormatter):
......@@ -15,29 +16,41 @@ class SortingHelpFormatter(HelpFormatter):
super(SortingHelpFormatter, self).add_arguments(actions)
def parse_arguments(description, add_run=False, add_irf=False, add_job=False, add_dl3=False):
def parse_arguments(description, add_dl2=False, add_irf=False, add_job=False, add_dl3=False, create_runlist=False, add_common=False):
parser = argparse.ArgumentParser(description=description, formatter_class=SortingHelpFormatter)
parser.add_argument('--prod', '-p',
dest='prod', required=False, type=str, default='v0.9.4',
help='Prod to use (default: %(default)s)')
parser.add_argument('--outdir', '-o',
dest='outdir', required=False, type=str, default=None,
help='Directory to store the output')
parser.add_argument('--config', '-c',
type=str, default=None, dest='config',
help='Specify a personal config file for the analysis')
parser.add_argument('--config-analysis',
type=str, default=None, dest='config_analysis',
help='Specify a config file which describes analysis profile to use')
# parser.add_argument('--config-analysis',
# type=str, default=None, dest='config_analysis',
# help='Specify a config file which describes analysis profile to use')
parser.add_argument('--verbose', '-v',
action='store_true', dest='verbose',
help='Increase output verbosity')
if add_run:
if create_runlist:
parser.add_argument('--source_name', '-n', required=True, default=None,
dest='source_name', type=str,
help='Name of the source')
parser.add_argument('--ra', type=float, dest='ra',
help='RA coordinate of the target. To add if you want to use custom position')
parser.add_argument('--dec', type=float, dest='dec',
help='Dec coordinate of the target. To add if you want to use custom position')
parser.add_argument('--night', type=str, dest='night',
nargs='+', required=False,
help='Night of the observation in the format YYYYMMDD. More nights can be specified')
if add_common:
parser.add_argument('--source_name', '-n', required=True,
default=None, dest='source_name', type=str,
help='Name of the source')
parser.add_argument('--config', '-c', type=str, default=None,
required=True, dest='config', help='Specify a personal config file for the analysis')
parser.add_argument('--dry',
default=False, required=False, action='store_true', dest='dry',
help='Make a dry run, no true submission')
if add_dl2:
parser.add_argument('--outdir', '-o', dest='outdir', required=False,
type=str, default=None, help='Directory to store the output')
parser.add_argument('--tcuname',
default=None, dest='tcuname', type=str,
help='Apply run selection based on TCU source name')
......@@ -46,9 +59,7 @@ def parse_arguments(description, add_run=False, add_irf=False, add_job=False, ad
help='File with a list of run and the associated night to be analysed')
parser.add_argument('--distance', '-dis',
type=float, dest='distance', default=-1,
help='Max distance in degrees between the target position and the run pointing \
position for the run selection, negative value means no selection using \
this parameter (default: %(default)s).')
help='Max distance in degrees between the target position and the run pointing position for the run selection. Negative value means no selection using this parameter (default: %(default)s).')
parser.add_argument('--ra',
type=float, dest='ra', default=-1,
help='RA coordinate of the target. To add if you want to use custom position')
......@@ -56,17 +67,6 @@ def parse_arguments(description, add_run=False, add_irf=False, add_job=False, ad
type=float, dest='dec', default=-91,
help='Dec coordinate of the target. To add if you want to use custom position')
if add_job:
parser.add_argument('--submit',
default=False, dest='submit', action='store_true', required=False,
help='Submit the cmd to slurm on site')
parser.add_argument('--dry',
default=False, required=False, action='store_true', dest='dry',
help='Make a dry run, no true submission')
parser.add_argument('--globber', '-g',
dest='globber', action='store_true', required=False, default=False,
help='If True, overwrites existing output file without asking')
if add_irf:
parser.add_argument('--gh_cut',
required=False, type=float, dest='gh_cut',
......@@ -79,6 +79,17 @@ def parse_arguments(description, add_run=False, add_irf=False, add_job=False, ad
help='Observation time for IRF in hours')
if add_dl3:
parser.add_argument('--outdir', '-o', dest='outdir', type=str,
default=None, help='Directory to store the output')
parser.add_argument('--runlist', '-rl',
default=None, dest='runlist', type=str,
help='File with a list of run and the associated night to be analysed')
parser.add_argument('--ra',
type=float, dest='ra', default=-1,
help='RA coordinate of the target. To add if you want to use custom position')
parser.add_argument('--dec',
type=float, dest='dec', default=-91,
help='Dec coordinate of the target. To add if you want to use custom position')
parser.add_argument('--cut_file', '-cut',
default=None, dest='cut_file', type=str,
help='Cut file')
......@@ -88,6 +99,14 @@ def parse_arguments(description, add_run=False, add_irf=False, add_job=False, ad
parser.add_argument('--theta_cut',
required=False, type=float, dest='theta_cut',
help='Fixed selection cut for theta')
if add_job:
parser.add_argument('--submit', default=False, dest='submit',
action='store_true', required=False, help='Submit the cmd to slurm on site')
parser.add_argument('--globber', '-g',
dest='globber', action='store_true', required=False, default=False,
help='If True, overwrites existing output file without asking (default: %(default)s).')
args = parser.parse_args()
return args
......@@ -137,7 +156,7 @@ def get_runs_database(args, database):
if args.tcuname is not None:
selection = database.loc[database['Target']].isin([args.tcuname])
if args.verbose:
print("Selection of runs based on the TCU name", args.tcuname, ". Only run with the name in the TCU are kept")
print("Selection of runs based on the TCU name", args.tcuname, ". Only runs with the name in the TCU are kept")
print(selection.index)
# if args.night[0] != 'all':
......@@ -156,12 +175,12 @@ def get_runs_database(args, database):
databaseRuns = np.array(selection.index)
if args.runlist is not None:
rl = np.loadtxt(args.runlist, unpack=True, dtype=int)
rl = np.loadtxt(os.environ.get('CONFIG_FOLDER') + '/' + args.runlist, unpack=True, dtype=int)
#databaseRuns = np.array([a for a in rl if a in databaseRuns])
databaseRuns = np.array([a for a in rl])
if args.verbose:
print("Final run selection", databaseRuns)
print("Final run selection: ", databaseRuns)
return databaseRuns
......@@ -257,7 +276,7 @@ def get_coordinates(args):
"""
returns the name and the Ra/Dec of the source
"""
success = False
if args.source_name is None:
raise ValueError("Please provide the name of the analysed source by using --source_name")
......@@ -267,13 +286,43 @@ def get_coordinates(args):
c = SkyCoord.from_name(args.source_name)
ra = c.ra.value
dec = c.dec.value
success = True
if args.verbose:
print("Coordinates of ", args.source_name, "found using Astropy: RA ", ra, ", Dec ", dec)
except:
print("Cannot resolve target name", args.source_name, "using Astropy. Switch to the Ra and Dec provided.")
if args.ra >= 0 and args.ra < 360 and args.dec >= -90 and args.dec < 90:
ra = args.ra
dec = args.dec
print("Using user provided RA and Dec.")
print("Cannot resolve target name", args.source_name, "using Astropy. Switch to the Ra and Dec provided by the user.")
if all(item is not None for item in [args.ra, args.dec]):
if args.ra >= 0 and args.ra < 360 and args.dec >= -90 and args.dec < 90:
ra = args.ra
dec = args.dec
if args.verbose:
print("Coordinates provided by the user: RA ", ra, ", Dec", dec)
else:
print("Please provide RA and Dec values by using --ra and --dec")
exit(0)
if (success is True and args.ra and args.dec):
if args.ra != ra or args.dec != dec:
print(f"WARNING! Astropy coordinates RA {ra}, Dec {dec} are different than the ones provided by the user RA {args.ra}, Dec {args.dec}.")
return ra, dec
def print_runs(table, mask, by_date=False):
"""
function to print out the run numbers that survive a certain set of cuts
"""
print(f"{mask.sum()} wobble runs for the selected source")
print(f"Observation time: {table['elapsed_time'][mask].sum()/3600:.2f} hours")
print()
print(np.array2string(np.array(table['runnumber'][mask]), separator=', '))
if by_date:
print()
print()
dates = [datetime.utcfromtimestamp(t - 0.5 * 86400).date() for t in table['time'][mask]]
for i, date in enumerate(np.unique(dates)):
rr = []
for d, run in zip(dates, table['runnumber'][mask]):
if d != date:
continue
rr.append(run)
print(i + 1, ":", date, ":", rr)