Unverified Commit e041ef71 authored by Ambra Di Piano's avatar Ambra Di Piano Committed by GitHub
Browse files

Merge pull request #2 from ambra-dipiano/simulator 

Update simulator
parents f8339748 fc1e0c9c

.gitignore

+3 −0
Original line number Diff line number Diff line 
# Patterns

.*

*.png

*.yml



# Folders

tmp/

tmp_*/

analysis_archive/



# Files
@@ -15,6 +17,7 @@ rtasci/rtasci/cfg/leo_bkg.yml 
!rtamock/execute_run.sh

!environment.yml

!rtavis/rtavis/lib/*py

!astrort/configure/test.yml



# Byte-compiled / optimized / DLL files

__pycache__/

astrort/configure/check_configuration.py

+36 −2
Original line number Diff line number Diff line
@@ -20,12 +20,12 @@ class CheckConfiguration(): 
        return self



    def check_tags(self):

        tags = ['simulator', 'visibility']

        tags = ['simulator', 'visibility', 'logging']

        assert self.conf.keys() == tags

        return self



    def check_simulator(self):

        keys = ['name', 'array', 'irf', 'prod', 'pointing', 'duration', 'samples', 'seed', 'model']

        keys = ['name', 'array', 'irf', 'prod', 'pointing', 'duration', 'samples', 'seed', 'model', 'output']

        assert self.conf['simulator'].keys() == keys

        assert type(self.conf['simulator']['name']) == str

        assert type(self.conf['simulator']['array']) in ['lst', 'mst', 'sst', 'cta', 'north', 'south']
@@ -36,6 +36,7 @@ class CheckConfiguration(): 
        assert type(self.conf['simulator']['samples']) == int

        assert type(self.conf['simulator']['seed']) == int

        assert type(self.conf['simulator']['model']) == str

        assert type(self.conf['simulator']['output']) == str

        return self



    def check_visibility(self):
@@ -43,3 +44,36 @@ class CheckConfiguration(): 
        assert self.conf['visibility'].keys() == keys

        assert type(self.conf['visibility']['start_time']) == str

        return self



    def check_logging(self):

        keys = ['level', 'logfile', 'datfile']

        assert self.conf['logging'].keys() == keys

        assert (type(self.conf['logging']['level']) == str or type(self.conf['logging']['level']) == int) 

        assert type(self.conf['logging']['logfile']) == str 

        assert type(self.conf['logging']['datfile']) == str 

        return self

    

    def check_slurm(self):

        keys = ['nodes', 'tasks', 'cpus', 'mem', 'environment', 'name', 'account', 'partition']

        assert self.conf['slurm'].keys() == keys

        assert type(self.conf['slurm']['nodes']) == int

        assert type(self.conf['slurm']['tasks']) == int

        assert type(self.conf['slurm']['cpus']) == int

        assert type(self.conf['slurm']['mem']) == str 

        assert type(self.conf['slurm']['environment']) == str 

        assert type(self.conf['slurm']['name']) == str 

        assert type(self.conf['slurm']['account']) == str 

        assert type(self.conf['slurm']['partition']) == str 

        return self

    

    def check_mapper(self):

        keys = ['exposure', 'smooth', 'pixelsize', 'center', 'plot', 'region', 'output']

        assert self.conf['mapper'].keys() == keys

        assert type(self.conf['mapper']['exposure']) == int

        assert type(self.conf['mapper']['smooth']) == (float or int)

        assert type(self.conf['mapper']['pixelsize']) == (float or int)

        assert type(self.conf['mapper']['center']) in ['pointing', 'source'] 

        assert type(self.conf['mapper']['plot']) == bool

        assert type(self.conf['mapper']['region']) == bool

        assert type(self.conf['mapper']['output']) == str

        return self
 
 No newline at end of file

astrort/configure/logging.py

0 → 100644
+53 −0
Original line number Diff line number Diff line 
# *****************************************************************************

# Copyright (C) 2023 INAF

# This software is distributed under the terms of the BSD-3-Clause license

#

# Authors:

# Ambra Di Piano <ambra.dipiano@inaf.it>

# *****************************************************************************



import logging

from os import makedirs

from os.path import dirname, isfile, join



def set_logger(level, filename=None):

    log = logging.getLogger()

    log.setLevel(level)

    # console logger

    formatter = logging.Formatter('[%(asctime)s] %(levelname)s: %(message)s')

    consoleHandler = logging.StreamHandler()

    consoleHandler.setFormatter(formatter)

    log.addHandler(consoleHandler)

    # file logger

    if filename is not None:

        makedirs(dirname(filename), exist_ok=True)

        if not isfile(filename):

            f = open(filename, 'w+')

            f.close()

        fileHandler = logging.FileHandler(filename)

        fileHandler.setFormatter(formatter)

        log.addHandler(fileHandler)

    return log



def get_log_level(level):

    if level in ['DEBUG', 'debug', 10]:

        level = logging.DEBUG

    elif level in ['INFO', 'info', 20]:

        level = logging.INFO

    elif level in ['WARN', 'WARNING', 'warn', 'warning', 30]:

        level = logging.WARN

    elif level in ['ERROR', 'error', 40]:

        level = logging.ERROR

    elif level in ['CRITICAL', 'critical', 50]:

        level = logging.CRITICAL

    else:

        level = logging.NOTSET

    return level



def get_logfile(configuration, mode):

    logfile = configuration['logging']['logfile']

    if mode not in logfile:

        logfile = logfile.replace('.log', f'_{mode}.log')

    if configuration[mode]['output'] not in logfile:

        logfile = join(configuration[mode]['output'], logfile)

    return logfile
 
 No newline at end of file

astrort/configure/slurmjobs.py

0 → 100644
+65 −0
Original line number Diff line number Diff line 
# *****************************************************************************

# Copyright (C) 2023 INAF

# This software is distributed under the terms of the BSD-3-Clause license

#

# Authors:

# Ambra Di Piano <ambra.dipiano@inaf.it>

# *****************************************************************************



from yaml import dump

from os import system, makedirs

from os.path import join, dirname, abspath

from astrort.utils.wrap import load_yaml_conf

from astrort.configure.logging import set_logger, get_log_level



def make_configuration(jobname_conf, configuration, node_number, mode):

    # simulator

    configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1

    # logging

    configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log')

    configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat')

    # write new configuration

    with open(jobname_conf, 'w+') as f:

        dump(configuration, f, default_flow_style=False)



def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode):

    # write sbatch

    with open(jobname_sh, 'w+') as f:

        f.write("#!/bin/bash")

        #f.write("\n#SBATCH --nodes=1")

        #f.write("\n#SBATCH --ntasks-per-node=1")

        #f.write("\n#SBATCH --cpus-per-task=2")

        f.write(f"\n#SBATCH --job-name={jobname}")

        #f.write(f"\n#SBATCH --mem={slurmconf['memory']}")

        f.write(f"\n#SBATCH --output={jobname_log}")

        f.write(f"\n#SBATCH --account={slurmconf['account']}")

        f.write(f"\n#SBATCH --partition={slurmconf['partition']}")

        f.write(f"\n")

        f.write(f"\nsource activate {slurmconf['environment']}")

        if mode == 'simulator':

            f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_simulator.py')} -f {jobname_conf}\n")

        elif mode == 'mapper':

            f.write(f"\npython {join(dirname(abspath(__file__)).replace('configure', 'simulator'), 'base_mapper.py')} -f {jobname_conf}\n")

        else:

            raise ValueError(f"Invalid 'mode' {mode}")



def make_sbatch(jobname, configuration, node_number, mode):

    output = configuration[mode]['output']

    jobname_sh = join(output, f"{jobname}_{mode}.sh")

    jobname_log = join(output, f"{jobname}_{mode}.slurm")

    jobname_conf = join(output, f"{jobname}_{mode}.yml")

    make_configuration(jobname_conf, configuration, node_number, mode=mode)

    make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode)

    system(f"sbatch {jobname_sh}")



def slurm_submission(configuration_file, nodes, mode):

    configuration = load_yaml_conf(configuration_file)

    log = set_logger(get_log_level(configuration['logging']['level']))

    # create output dir

    log.info(f"Creating {configuration[mode]['output']}")

    makedirs(configuration[mode]['output'], exist_ok=True)

    # sbatch jobs per each nodes

    configuration['slurm']['nodes'] = nodes

    for node_number in range(configuration['slurm']['nodes']):

        jobname = f"{configuration['slurm']['name']}_{node_number+1}"

        make_sbatch(jobname, configuration, node_number, mode=mode)

astrort/configure/test.yml

+29 −5
Original line number Diff line number Diff line 
simulator:

  name: test

  name: crab

  array: lst

  irf: North_z60_0.5h_LST

  prod: prod5

  prod: prod5-v0.1

  pointing: random

  duration: 300

  samples: 5

  duration: 10

  samples: 2

  seed: 1

  model: $TEMPLATES$/crab.xml

  output: /data01/homes/dipiano/astroRT/

  output: /data01/homes/dipiano/astroRT/astrort/testing/tmp

  

mapper:

  exposure: 10

  smooth: 1

  pixelsize: 0.02

  center: pointing

  plot: true

  region: false

  output: /data01/homes/dipiano/astroRT/astrort/testing/tmp



visibility:

  start_time: '2030-01-01T00:00:00'



logging:

  level: CRITICAL

  logfile: test.log

  datfile: test.dat



slurm:

  nodes: 5

  tasks: 1

  cpus: 2

  mem: 32G

  environment: astrort

  name: test

  account: dipiano

  partition: large