Unverified Commit 906cd2ce authored by Ambra Di Piano's avatar Ambra Di Piano Committed by GitHub
Browse files

simulator update (#6) 

* add mpthreads to argparse

* randomise pointing and test

* test randomise source

* replicate randomise source

* update test conf

* fix tests and bugs

* update

* remove unused

* fix bugs

* update check conf

* update

* remuve unused

* create maps for cleaner cnn

* rename

* skip galactic plane

* add script

* formatting up to threshold

* update

* fix replica

* fix folders

* add test

* fix irf
parent 1422f8f1

astrort/configure/check_configuration.py

+6 −2
Original line number Original line Diff line number Diff line
@@ -25,19 +25,21 @@ class CheckConfiguration(): 
        return self

        return self





    def check_simulator(self):

    def check_simulator(self):

        keys = ['name', 'array', 'irf', 'prod', 'pointing', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output']

        keys = ['name', 'array', 'irf', 'prod', 'pointing', 'target', 'maxoffset', 'duration', 'samples', 'seed', 'model', 'output', 'replicate']

        assert self.conf['simulator'].keys() == keys

        assert self.conf['simulator'].keys() == keys

        assert type(self.conf['simulator']['name']) == str

        assert type(self.conf['simulator']['name']) == str

        assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south']

        assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south']

        assert type(self.conf['simulator']['irf']) == str

        assert type(self.conf['simulator']['irf']) == str

        assert type(self.conf['simulator']['prod']) == str

        assert type(self.conf['simulator']['prod']) == str

        assert type(self.conf['simulator']['pointing']) == (str or dict)

        assert type(self.conf['simulator']['pointing']) == (str or dict)

        assert type(self.conf['simulator']['target']) == (str or None)

        assert type(self.conf['simulator']['maxoffset']) == (int or float)

        assert type(self.conf['simulator']['maxoffset']) == (int or float)

        assert type(self.conf['simulator']['duration']) == int

        assert type(self.conf['simulator']['duration']) == int

        assert type(self.conf['simulator']['samples']) == int

        assert type(self.conf['simulator']['samples']) == int

        assert type(self.conf['simulator']['seed']) == int

        assert type(self.conf['simulator']['seed']) == int

        assert type(self.conf['simulator']['model']) == str

        assert type(self.conf['simulator']['model']) == str

        assert type(self.conf['simulator']['output']) == str

        assert type(self.conf['simulator']['output']) == str

        assert type(self.conf['simulator']['replicate']) == (str or None)

        return self

        return self





    def check_visibility(self):

    def check_visibility(self):
@@ -68,7 +70,7 @@ class CheckConfiguration(): 
        return self

        return self

    

    

    def check_mapper(self):

    def check_mapper(self):

        keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output']

        keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output', 'replicate', 'save']

        assert self.conf['mapper'].keys() == keys

        assert self.conf['mapper'].keys() == keys

        assert type(self.conf['mapper']['exposure']) == int

        assert type(self.conf['mapper']['exposure']) == int

        assert type(self.conf['mapper']['smooth']) == (float or int)

        assert type(self.conf['mapper']['smooth']) == (float or int)
@@ -77,4 +79,6 @@ class CheckConfiguration(): 
        assert type(self.conf['mapper']['plot']) == bool

        assert type(self.conf['mapper']['plot']) == bool

        assert type(self.conf['mapper']['region']) == bool

        assert type(self.conf['mapper']['region']) == bool

        assert type(self.conf['mapper']['output']) == str

        assert type(self.conf['mapper']['output']) == str

        assert type(self.conf['simulator']['replicate']) == (str or None)

        assert type(self.conf['simulator']['save']) == str

        return self
 
        return self
 
 No newline at end of file

astrort/configure/slurmjobs.py

+9 −7
Original line number Original line Diff line number Diff line
@@ -19,12 +19,12 @@ def make_configuration(jobname_conf, configuration, node_number, mode): 
    configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log')

    configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log')

    configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat')

    configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat')

    if configuration[mode]['replicate'] is not None:

    if configuration[mode]['replicate'] is not None:

        configuration[mode]['replicate'] = join(dirname(configuration[mode]['replicate']), f'job_{node_number+1}_{mode}.dat')

        configuration[mode]['replicate'] = join(dirname(configuration[mode]['replicate']), f'job_{node_number+1}_simulator.dat')

    # write new configuration

    # write new configuration

    with open(jobname_conf, 'w+') as f:

    with open(jobname_conf, 'w+') as f:

        dump(configuration, f, default_flow_style=False)

        dump(configuration, f, default_flow_style=False)





def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode):

def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode, script=None):

    # write sbatch

    # write sbatch

    with open(jobname_sh, 'w+') as f:

    with open(jobname_sh, 'w+') as f:

        f.write("#!/bin/bash")

        f.write("#!/bin/bash")
@@ -40,21 +40,23 @@ def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode): 
        f.write(f"\nsource activate {slurmconf['environment']}")

        f.write(f"\nsource activate {slurmconf['environment']}")

        if mode == 'simulator':

        if mode == 'simulator':

            f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n")

            f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n")

        elif mode == 'mapper':

        elif mode == 'mapper' and script is None:

            f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_mapper.py')} -f {jobname_conf}\n")

            f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_mapper.py')} -f {jobname_conf}\n")

        elif mode == 'mapper' and script is not None:

            f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), f'{script}.py')} -f {jobname_conf}\n")

        else:

        else:

            raise ValueError(f"Invalid 'mode' {mode}")

            raise ValueError(f"Invalid 'mode' {mode}")





def make_sbatch(jobname, configuration, node_number, mode):

def make_sbatch(jobname, configuration, node_number, mode, script=None):

    output = configuration[mode]['output']

    output = configuration[mode]['output']

    jobname_sh = join(output, f"{jobname}_{mode}.sh")

    jobname_sh = join(output, f"{jobname}_{mode}.sh")

    jobname_log = join(output, f"{jobname}_{mode}.slurm")

    jobname_log = join(output, f"{jobname}_{mode}.slurm")

    jobname_conf = join(output, f"{jobname}_{mode}.yml")

    jobname_conf = join(output, f"{jobname}_{mode}.yml")

    make_configuration(jobname_conf, configuration, node_number, mode=mode)

    make_configuration(jobname_conf, configuration, node_number, mode=mode)

    make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode)

    make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode, script=script)

    system(f"sbatch {jobname_sh}")

    system(f"sbatch {jobname_sh}")





def slurm_submission(configuration_file, nodes, mode):

def slurm_submission(configuration_file, nodes, mode, script=None):

    configuration = load_yaml_conf(configuration_file)

    configuration = load_yaml_conf(configuration_file)

    log = set_logger(get_log_level(configuration['logging']['level']))

    log = set_logger(get_log_level(configuration['logging']['level']))

    # create output dir

    # create output dir
@@ -64,4 +66,4 @@ def slurm_submission(configuration_file, nodes, mode): 
    configuration['slurm']['nodes'] = nodes

    configuration['slurm']['nodes'] = nodes

    for node_number in range(configuration['slurm']['nodes']):

    for node_number in range(configuration['slurm']['nodes']):

        jobname = f"{configuration['slurm']['name']}_{node_number+1}"

        jobname = f"{configuration['slurm']['name']}_{node_number+1}"

        make_sbatch(jobname, configuration, node_number, mode=mode)
 
        make_sbatch(jobname, configuration, node_number, mode=mode, script=script)

astrort/configure/test.yml

+3 −0
Original line number Original line Diff line number Diff line
@@ -4,6 +4,7 @@ simulator: 
  irf: North_z60_0.5h_LST

  irf: North_z60_0.5h_LST

  prod: prod5-v0.1

  prod: prod5-v0.1

  pointing: random

  pointing: random

  target: null

  maxoffset: 2

  maxoffset: 2

  duration: 10

  duration: 10

  samples: 2

  samples: 2
@@ -20,8 +21,10 @@ mapper: 
  plot: true

  plot: true

  region: false

  region: false

  output: /data01/homes/dipiano/astroRT/astrort/testing/tmp

  output: /data01/homes/dipiano/astroRT/astrort/testing/tmp

  replicate: null

  save: npy

  save: npy







visibility:

visibility:

  start_time: '2030-01-01T00:00:00'

  start_time: '2030-01-01T00:00:00'

astrort/simulator/base_cleaner.py

0 → 100644
+99 −0
Original line number Original line Diff line number Diff line 
# *****************************************************************************

# Copyright (C) 2023 INAF

# This software is distributed under the terms of the BSD-3-Clause license

#

# Authors:

# Ambra Di Piano <ambra.dipiano@inaf.it>

# *****************************************************************************



import argparse

import pandas as pd

from time import time

from os import makedirs

from os.path import join

from astrort.utils.wrap import load_yaml_conf, write_mapping_info, plot_map

from astrort.utils.utils import get_all_seeds, get_instrument_fov, get_instrument_tev_range, adjust_tev_range_to_irf

from astrort.configure.logging import set_logger, get_log_level, get_logfile

from astrort.configure.slurmjobs import slurm_submission

from rtasci.lib.RTACtoolsAnalysis import RTACtoolsAnalysis



def base_cleaner(configuration_file, seeds=None):

    clock = time()

    configuration = load_yaml_conf(configuration_file)

    logfile = get_logfile(configuration, mode='mapper')

    datfile = logfile.replace('.log', '.dat')

    # set logger

    log = set_logger(get_log_level(configuration['logging']['level']), logfile)

    # collect simulations to map

    if seeds is None:

        log.info(f"Mapping of all simulations found")

        seeds = get_all_seeds(configuration['simulator'])

    log.info(f"Mapper configured, took {time() - clock} s")

    # create output dir

    log.info(f"Output folder: {configuration['mapper']['output']}")

    makedirs(configuration['mapper']['output'], exist_ok=True)

    makedirs(join(configuration['mapper']['output'], 'clean'), exist_ok=True)

    makedirs(join(configuration['mapper']['output'], 'noisy'), exist_ok=True)

    # start mapping

    log.info(f"\n {'-'*15} \n| START MAPPER | \n {'-'*15} \n")

    if configuration['mapper']['replicate'] is not None:

        replica = pd.read_csv(configuration['mapper']['replicate'], sep=' ', header=0)

        log.info(f"Replicate pointing and IRF from {configuration['mapper']['replicate']}")

    else:

        raise ValueError(f'This script requires a "mapper:replicate" configuration other than None')

    for seed in seeds:

        clock_map = time()

        # configure map

        mapper = RTACtoolsAnalysis()

        mapper.input = join(configuration['simulator']['output'], replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '.fits')

        log.debug(f"DL3: {mapper.input}")

        mapper.irf = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0] 

        log.debug(f"IRF: {mapper.irf}")

        mapper.caldb = configuration['simulator']['prod']

        mapper.e = adjust_tev_range_to_irf(get_instrument_tev_range(configuration['simulator']['array']), mapper.irf)

        mapper.t = [0, configuration['mapper']['exposure']]

        mapper.roi = get_instrument_fov(configuration['simulator']['array'])

        # make noisy map

        mapper.sky_subtraction = 'NONE'

        mapper.output = join(configuration['mapper']['output'], 'noisy', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits')

        mapper.run_skymap(wbin=configuration['mapper']['pixelsize'])

        log.debug(f"Noisy map: {mapper.output}")

        # make plot

        if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits':

            clock_plot = time()

            plotmap = plot_map(mapper.output, log)

            log.info(f"Plotting noisy image (seed = {seed}) complete, took {time() - clock_plot} s")

        # make clean map

        mapper.sky_subtraction = 'IRF'

        mapper.output = join(configuration['mapper']['output'], 'clean', replica[replica['seed']==configuration['simulator']['seed']]['name'].values[0] + '_map.fits')

        mapper.run_skymap(wbin=configuration['mapper']['pixelsize'])

        log.debug(f"Clean map: {mapper.output}")

        # make plot

        if configuration['mapper']['plot'] and configuration['mapper']['save'] == 'fits':

            clock_plot = time()

            plotmap = plot_map(mapper.output, log)

            log.info(f"Plotting clean image (seed = {seed}) complete, took {time() - clock_plot} s")

        del mapper

        log.info(f"Mapping (seed = {seed}) complete, took {time() - clock_map} s")

        # timing simulation

        clock_map = time() - clock_map

        # save simulation data

        write_mapping_info(configuration, datfile, clock_map)

        configuration['simulator']['seed'] += 1

    # end simulations

    log.info(f"\n {'-'*15} \n| STOP MAPPER | \n {'-'*15} \n")

    log.info(f"Process complete, took {time() - clock} s")



def main(configuration, nodes):

    if nodes == 0:

        base_cleaner(configuration)

    else:

        slurm_submission(configuration, nodes, mode='mapper', script='base_cleaner')



if __name__ == '__main__':

    parser = argparse.ArgumentParser(description='')

    parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file")

    parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' )

    args = parser.parse_args()



    main(args.configuration, args.nodes)
 
 No newline at end of file

astrort/simulator/base_simulator.py

+7 −2
Original line number Original line Diff line number Diff line
@@ -10,7 +10,7 @@ import argparse 
import pandas as pd

import pandas as pd

from time import time

from time import time

from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation

from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation

from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf

from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing, set_irf, randomise_target, replicate_target

from astrort.configure.logging import set_logger, get_log_level, get_logfile

from astrort.configure.logging import set_logger, get_log_level, get_logfile

from astrort.configure.slurmjobs import slurm_submission

from astrort.configure.slurmjobs import slurm_submission
@@ -23,7 +23,6 @@ def base_simulator(configuration_file): 
    log.info(f"Simulator configured, took {time() - clock} s")

    log.info(f"Simulator configured, took {time() - clock} s")

    # create output dir

    # create output dir

    log.info(f"Output folder: {configuration['simulator']['output']}")

    log.info(f"Output folder: {configuration['simulator']['output']}")

    #makedirs(configuration['simulator']['output'], exist_ok=True)

    # start simulations

    # start simulations

    log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n")

    log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n")

    if configuration['simulator']['replicate'] is not None:

    if configuration['simulator']['replicate'] is not None:
@@ -34,12 +33,17 @@ def base_simulator(configuration_file): 
    # loop seeds

    # loop seeds

    for i in range(configuration['simulator']['samples']):

    for i in range(configuration['simulator']['samples']):

        clock_sim = time()

        clock_sim = time()

        # randomise source position in model

        if configuration['simulator']['target'] == 'random' and replica is None:

            configuration['simulator']['model'] = randomise_target(model=configuration['simulator']['model'], output=configuration['simulator']['output'], name=configuration['simulator']['name'], samples=configuration['simulator']['samples'], seed=configuration['simulator']['seed'])

        simulator = RTACtoolsSimulation()

        simulator = RTACtoolsSimulation()

        # check pointing option

        # check pointing option

        if replica is not None:

        if replica is not None:

            configuration['simulator']['pointing'] = {'ra': replica[replica['seed']==configuration['simulator']['seed']]['point_ra'].values[0],  

            configuration['simulator']['pointing'] = {'ra': replica[replica['seed']==configuration['simulator']['seed']]['point_ra'].values[0],  

                                                      'dec': replica[replica['seed']==configuration['simulator']['seed']]['point_dec'].values[0]}

                                                      'dec': replica[replica['seed']==configuration['simulator']['seed']]['point_dec'].values[0]}

            configuration['simulator']['irf'] = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0]   

            configuration['simulator']['irf'] = replica[replica['seed']==configuration['simulator']['seed']]['irf'].values[0]   

            configuration['simulator']['model'] = replicate_target(model=configuration['simulator']['model'], output=configuration['simulator']['output'], name=configuration['simulator']['name'], samples=configuration['simulator']['samples'], seed=configuration['simulator']['seed'], ra=replica[replica['seed']==configuration['simulator']['seed']]['source_ra'].values[0], dec=replica[replica['seed']==configuration['simulator']['seed']]['source_dec'].values[0])



        simulator, point = set_pointing(simulator, configuration['simulator'], log)

        simulator, point = set_pointing(simulator, configuration['simulator'], log)

        simulator.irf = set_irf(configuration['simulator'], log)

        simulator.irf = set_irf(configuration['simulator'], log)

        # complete configuration

        # complete configuration
@@ -66,6 +70,7 @@ if __name__ == '__main__': 
    parser = argparse.ArgumentParser(description='')

    parser = argparse.ArgumentParser(description='')

    parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file")

    parser.add_argument('-f', '--configuration', type=str, required=True, help="Path of yaml configuration file")

    parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' )

    parser.add_argument('-n', '--nodes', type=int, default=0, help='Number of slurm nodes to occupy for submission, if unset it will not submit to slurm' )

    parser.add_argument('-mp', '--mpthreads', type=int, default=0, choices=range(0,7), help='Number of threads to use for parallel simulation, if unset it will not use multi-threading' )

    args = parser.parse_args()

    args = parser.parse_args()





    main(args.configuration, args.nodes)

    main(args.configuration, args.nodes)