Skip to content
GitLab
Explore
Sign in
Alice Donini
lst-analysis
Compare revisions
66cb0a27a7916ec6ffbfaf578ab87288388eb4d4 to 941e3ce442fb77bfab906f32c38ce3e2baf569cf
Hide whitespace changes
Inline
Side-by-side
docs/source/usage/index_generation.rst
0 → 100644
View file @
941e3ce4
.. _index_generation:
Index creation
==============
Gammapy needs an index file to read the generated DL3 fits file (:ref:`dl3`). This can be created using the `create_DL3_index.py` script.
The script makes use of the same configuration file of the script to generate DL3 files: `config_dl2_to_dl3.yaml`.
Usage:
.. code-block:: bash
usage: create_DL3_index.py [-h] [--prod PROD] [--outdir OUTDIR] [--config CONFIG]
[--config-analysis CONFIG_ANALYSIS] [--verbose] [--source_name SOURCE_NAME]
[--tcuname TCUNAME] [--runlist RUNLIST [RUNLIST ...]] [--distance DISTANCE]
[--ra RA] [--dec DEC]
DL3 index maker
optional arguments:
--config CONFIG, -c CONFIG
Specify a personal config file for the analysis
--config-analysis CONFIG_ANALYSIS
Specify a config file which described analysis profile to use
--dec DEC Dec coordinate of the target. To add if you want to use custom position
--distance DISTANCE, -dis DISTANCE
Max distance in degrees between the target position and the run pointing position
for the run selection, negative value means no selection using this parameter
(default: -1).
--outdir OUTDIR, -o OUTDIR
Directory to store the output
--prod PROD, -p PROD Prod to use (default: v0.8.4)
--ra RA RA coordinate of the target. To add if you want to use custom position
--runlist RUNLIST [RUNLIST ...], -rl RUNLIST [RUNLIST ...]
File with run list to be analysed
--source_name SOURCE_NAME, -n SOURCE_NAME
Name of the source
--tcuname TCUNAME Apply run selection based on TCU source name
--verbose, -v Increase output verbosity
-h, --help show this help message and exit
The command is pretty similar to the one for the DL3 generation. The code is fast, so no submission process has been implemented.
.. code-block:: bash
python create_DL3_index.py -c config_dl2_to_dl3_Crab2020.yaml --source_name Crab -v
After running the script, you should have two files in the DL3 data directory: `hdu-index.fits.gz` and `obs-index.fits.gz`.
.. note::
The index files have to be in the same folder of the related DL3 file.
init.sh
0 → 100644
View file @
941e3ce4
# To activate the pre-installed conda env and the latest lstchain env, uncomment the following two lines
#source /fefs/aswg/software/conda/etc/profile.d/conda.sh
#conda activate lstchain-v0.9.1
# personal lstchain conda env comment if you want to use the pre-installed one
conda activate lst-dev
export
CODE_DIR
=
/../lst_scripts
export
PYTHONPATH
=
$CODE_DIR
:
$PYTHONPATH
export
CONFIG_FOLDER
=
../
utils.py
0 → 100644
View file @
941e3ce4
import
argparse
import
os
import
pandas
as
pd
import
numpy
as
np
import
yaml
from
operator
import
attrgetter
from
astropy.coordinates
import
SkyCoord
from
glob
import
glob
from
argparse
import
HelpFormatter
class
SortingHelpFormatter
(
HelpFormatter
):
def
add_arguments
(
self
,
actions
):
actions
=
sorted
(
actions
,
key
=
attrgetter
(
'
option_strings
'
))
super
(
SortingHelpFormatter
,
self
).
add_arguments
(
actions
)
def
parse_arguments
(
description
,
add_run
=
False
,
add_irf
=
False
,
add_job
=
False
,
add_dl3
=
False
):
parser
=
argparse
.
ArgumentParser
(
description
=
description
,
formatter_class
=
SortingHelpFormatter
)
parser
.
add_argument
(
'
--prod
'
,
'
-p
'
,
dest
=
'
prod
'
,
required
=
False
,
type
=
str
,
default
=
'
v0.9.4
'
,
help
=
'
Prod to use (default: %(default)s)
'
)
parser
.
add_argument
(
'
--outdir
'
,
'
-o
'
,
dest
=
'
outdir
'
,
required
=
False
,
type
=
str
,
default
=
None
,
help
=
'
Directory to store the output
'
)
parser
.
add_argument
(
'
--config
'
,
'
-c
'
,
type
=
str
,
default
=
None
,
dest
=
'
config
'
,
help
=
'
Specify a personal config file for the analysis
'
)
parser
.
add_argument
(
'
--config-analysis
'
,
type
=
str
,
default
=
None
,
dest
=
'
config_analysis
'
,
help
=
'
Specify a config file which describes analysis profile to use
'
)
parser
.
add_argument
(
'
--verbose
'
,
'
-v
'
,
action
=
'
store_true
'
,
dest
=
'
verbose
'
,
help
=
'
Increase output verbosity
'
)
if
add_run
:
parser
.
add_argument
(
'
--source_name
'
,
'
-n
'
,
required
=
True
,
default
=
None
,
dest
=
'
source_name
'
,
type
=
str
,
help
=
'
Name of the source
'
)
parser
.
add_argument
(
'
--tcuname
'
,
default
=
None
,
dest
=
'
tcuname
'
,
type
=
str
,
help
=
'
Apply run selection based on TCU source name
'
)
parser
.
add_argument
(
'
--runlist
'
,
'
-rl
'
,
default
=
None
,
dest
=
'
runlist
'
,
type
=
str
,
help
=
'
File with a list of run and the associated night to be analysed
'
)
parser
.
add_argument
(
'
--distance
'
,
'
-dis
'
,
type
=
float
,
dest
=
'
distance
'
,
default
=-
1
,
help
=
'
Max distance in degrees between the target position and the run pointing
\
position for the run selection, negative value means no selection using
\
this parameter (default: %(default)s).
'
)
parser
.
add_argument
(
'
--ra
'
,
type
=
float
,
dest
=
'
ra
'
,
default
=-
1
,
help
=
'
RA coordinate of the target. To add if you want to use custom position
'
)
parser
.
add_argument
(
'
--dec
'
,
type
=
float
,
dest
=
'
dec
'
,
default
=-
91
,
help
=
'
Dec coordinate of the target. To add if you want to use custom position
'
)
if
add_job
:
parser
.
add_argument
(
'
--submit
'
,
default
=
False
,
dest
=
'
submit
'
,
action
=
'
store_true
'
,
required
=
False
,
help
=
'
Submit the cmd to slurm on site
'
)
parser
.
add_argument
(
'
--dry
'
,
default
=
False
,
required
=
False
,
action
=
'
store_true
'
,
dest
=
'
dry
'
,
help
=
'
Make a dry run, no true submission
'
)
parser
.
add_argument
(
'
--globber
'
,
'
-g
'
,
dest
=
'
globber
'
,
action
=
'
store_true
'
,
required
=
False
,
default
=
False
,
help
=
'
If True, overwrites existing output file without asking
'
)
if
add_irf
:
parser
.
add_argument
(
'
--gh_cut
'
,
required
=
False
,
type
=
float
,
dest
=
'
gh_cut
'
,
help
=
'
Fixed selection cut for gh_score (gammaness)
'
)
parser
.
add_argument
(
'
--theta_cut
'
,
required
=
False
,
type
=
float
,
dest
=
'
theta_cut
'
,
help
=
'
Fixed selection cut for theta
'
)
parser
.
add_argument
(
'
--obs_time
'
,
required
=
False
,
type
=
float
,
dest
=
'
obs_time
'
,
help
=
'
Observation time for IRF in hours
'
)
if
add_dl3
:
parser
.
add_argument
(
'
--cut_file
'
,
'
-cut
'
,
default
=
None
,
dest
=
'
cut_file
'
,
type
=
str
,
help
=
'
Cut file
'
)
parser
.
add_argument
(
'
--gh_cut
'
,
required
=
False
,
type
=
float
,
dest
=
'
gh_cut
'
,
help
=
'
Fixed selection cut for gh_score (gammaness)
'
)
parser
.
add_argument
(
'
--theta_cut
'
,
required
=
False
,
type
=
float
,
dest
=
'
theta_cut
'
,
help
=
'
Fixed selection cut for theta
'
)
args
=
parser
.
parse_args
()
return
args
def
get_db
(
database_filename
):
"""
Parameters
----------
database_filename : name of database file
---------
Returns the DB
"""
db_file
=
os
.
environ
.
get
(
'
CONFIG_FOLDER
'
)
+
'
/
'
+
database_filename
database
=
pd
.
read_csv
(
db_file
,
index_col
=
0
,
parse_dates
=
True
)
return
database
def
get_config
(
config_filename
):
"""
Parameters
----------
config_filename : name of the configuration file
---------
Returns the configuration file
"""
config_file
=
os
.
environ
.
get
(
'
CONFIG_FOLDER
'
)
+
'
/
'
+
config_filename
with
open
(
config_file
)
as
f
:
config_analysis
=
yaml
.
load
(
f
,
Loader
=
yaml
.
FullLoader
)
return
config_analysis
def
get_runs_database
(
args
,
database
):
# make the run list into a np array
databaseRuns
=
np
.
array
(
database
.
index
)
# Apply selection of data if argument is specified. Based on tcuname, run or coordinates
selection
=
database
#if args.tcuname[0] == 'all' and args.night[0] == 'all' and args.runlist[0] == 'none':
# raise RuntimeError("Cannont make a run selection. Either tcuname, night or runlist or distance is needed")
if
args
.
tcuname
is
not
None
:
selection
=
database
.
loc
[
database
[
'
Target
'
]].
isin
([
args
.
tcuname
])
if
args
.
verbose
:
print
(
"
Selection of runs based on the TCU name
"
,
args
.
tcuname
,
"
. Only run with the name in the TCU are kept
"
)
print
(
selection
.
index
)
# if args.night[0] != 'all':
# selection = database.loc[database['day'].isin(args.night)]
# if args.verbose:
# print("selection of night", args.night[0])
# print(selection.index)
if
args
.
distance
>
0
:
if
args
.
ra
==
-
1
or
args
.
dec
==
-
91
:
raise
RuntimeError
(
"
Cannont make a run selection. Ra and Dec value of the source are not given.
"
)
selection
=
database
.
loc
[((
database
[
'
RA_Obs
'
]
-
args
.
ra
)
**
2
+
(
database
[
'
Dec_Obs
'
]
-
args
.
dec
)
**
2
)
<
args
.
distance
**
2
]
if
args
.
verbose
:
print
(
"
Selection based on angular distance
"
,
args
.
distance
,
"
°
"
)
print
(
selection
.
index
)
databaseRuns
=
np
.
array
(
selection
.
index
)
if
args
.
runlist
is
not
None
:
rl
=
np
.
loadtxt
(
args
.
runlist
,
unpack
=
True
,
dtype
=
int
)
#databaseRuns = np.array([a for a in rl if a in databaseRuns])
databaseRuns
=
np
.
array
([
a
for
a
in
rl
])
if
args
.
verbose
:
print
(
"
Final run selection
"
,
databaseRuns
)
return
databaseRuns
def
create_DL1_list
(
args
,
config
,
runs
,
night
):
"""
create list with all the DL1 run path to analyze
Parameters
----------
runs: List of run numbers
night: night to analyze
"""
version
=
config
[
'
dl1_data
'
][
'
version
'
]
cleaning
=
config
[
'
dl1_data
'
][
'
cleaning
'
]
folder
=
config
[
'
data_folder
'
]
+
'
/DL1/
'
+
night
+
'
/
'
+
version
+
'
/
'
+
cleaning
+
'
/dl1*
'
# Current format of LST data path
if
args
.
verbose
:
print
(
"
Looking for files here :
"
,
folder
)
filepath_glob
=
glob
(
folder
)
# initializa data frame
filelist
=
pd
.
DataFrame
(
columns
=
[
'
path
'
,
'
night
'
])
# Create a list of files with matching run numbers
newEntry
=
{}
for
filename
in
filepath_glob
:
for
run
in
runs
:
if
(
f
"
Run
{
run
:
>
05
}
.h5
"
in
filename
):
newEntry
[
'
path
'
]
=
filename
newEntry
[
'
night
'
]
=
night
filelist
.
loc
[
run
]
=
newEntry
[
'
path
'
],
newEntry
[
'
night
'
]
return
filelist
def
create_DL2_list
(
args
,
config
,
runs
,
night
):
"""
create list with all the DL2 run path to analyze
Parameters
----------
runs: List of run numbers
"""
version
=
config
[
'
dl2_data
'
][
'
version
'
]
cleaning
=
config
[
'
dl2_data
'
][
'
cleaning
'
]
folder
=
config
[
'
data_folder
'
]
+
'
/DL2/
'
+
night
+
'
/
'
+
version
+
'
/
'
+
cleaning
+
'
/dl2*
'
# Current format of LST data path
if
args
.
verbose
:
print
(
"
looking for files here :
"
,
folder
)
filepath_glob
=
glob
(
folder
)
filelist
=
pd
.
DataFrame
(
columns
=
[
'
path
'
])
# Create a list of files with matching run numbers
newEntry
=
{}
for
filename
in
filepath_glob
:
for
run
in
runs
:
if
(
f
"
Run
{
run
:
>
05
}
.h5
"
in
filename
):
newEntry
[
'
path
'
]
=
filename
filelist
.
loc
[
run
]
=
newEntry
return
filelist
def
manage_submission
(
args
,
config
,
cmd
,
run
,
level
=
"
3
"
):
"""
Parameters
----------
config: personal configuration file with analysis settings
cmd : command to be run
run : run number
"""
print
(
"
Submission of the jobs to the slurm farm.
"
)
os
.
makedirs
(
config
[
'
jobmanager
'
],
exist_ok
=
True
)
template
=
open
(
os
.
environ
.
get
(
'
CODE_DIR
'
)
+
"
/SubmitTemplate_dl
"
+
level
+
"
.sh
"
,
"
r
"
).
readlines
()
scriptname
=
config
[
'
jobmanager
'
]
+
"
/Script_dl
"
+
level
+
"
_
"
+
str
(
run
)
+
"
.sh
"
logfile
=
config
[
'
jobmanager
'
]
+
"
/Slurm_dl
"
+
level
+
"
_
"
+
str
(
run
)
+
"
.out
"
script
=
open
(
scriptname
,
"
w
"
)
for
t
in
template
:
script
.
write
(
t
.
replace
(
"
jobname
"
,
"
Job_DL
"
+
level
+
"
_
"
+
f
"
Run
{
run
:
>
05
}
"
).
replace
(
"
logfile
"
,
logfile
))
script
.
write
(
"
\n
"
)
script
.
write
(
cmd
)
script
.
close
()
os
.
system
(
"
chmod +x
"
+
scriptname
)
return
scriptname
def
get_coordinates
(
args
):
"""
returns the name and the Ra/Dec of the source
"""
if
args
.
source_name
is
None
:
raise
ValueError
(
"
Please provide the name of the analysed source by using --source_name
"
)
if
args
.
verbose
:
print
(
"
Search Coordinates for
"
,
args
.
source_name
,
"
using Astropy and based on the target name.
"
)
try
:
c
=
SkyCoord
.
from_name
(
args
.
source_name
)
ra
=
c
.
ra
.
value
dec
=
c
.
dec
.
value
except
:
print
(
"
Cannot resolve target name
"
,
args
.
source_name
,
"
using Astropy. Switch to the Ra and Dec provided.
"
)
if
args
.
ra
>=
0
and
args
.
ra
<
360
and
args
.
dec
>=
-
90
and
args
.
dec
<
90
:
ra
=
args
.
ra
dec
=
args
.
dec
print
(
"
Using user provided RA and Dec.
"
)
else
:
print
(
"
Please provide RA and Dec values by using --ra and --dec
"
)
exit
(
0
)
return
ra
,
dec
Prev
1
2
Next