Commit 628d53d6 authored by Ambra Di Piano's avatar Ambra Di Piano
Browse files

fix makedirs and rmtree and also logname

parent 3d9eac84

astrort/configure/logging.py

+9 −4
Original line number Diff line number Diff line
@@ -8,7 +8,7 @@ 


import logging

from os import makedirs

from os.path import isdir, dirname, isfile

from os.path import dirname, isfile, join



def set_logger(level, filename=None):

    log = logging.getLogger()
@@ -20,8 +20,7 @@ def set_logger(level, filename=None): 
    log.addHandler(consoleHandler)

    # file logger

    if filename is not None:

        if not isdir(dirname(filename)):

            makedirs(dirname(filename))

        makedirs(dirname(filename), exist_ok=True)

        if not isfile(filename):

            f = open(filename, 'w+')

            f.close()
@@ -45,4 +44,10 @@ def get_log_level(level): 
        level = logging.NOTSET

    return level



    return
 
 No newline at end of file
 
def get_logfile(configuration, mode):

    logfile = configuration['logging']['logfile']

    if mode not in logfile:

        logfile = logfile.replace('.log', f'_{mode}.log')

    if configuration[mode]['output'] not in logfile:

        logfile = join(configuration[mode]['output'], logfile)

    return logfile
 
 No newline at end of file

astrort/configure/slurmjobs.py

+7 −7
Original line number Diff line number Diff line
@@ -12,17 +12,17 @@ from os.path import join, dirname, abspath 
from astrort.utils.wrap import load_yaml_conf

from astrort.configure.logging import set_logger, get_log_level



def make_configuration(jobname_conf, configuration, node_number):

def make_configuration(jobname_conf, configuration, node_number, mode):

    # simulator

    configuration['simulator']['seed'] = node_number*configuration['simulator']['samples'] + 1

    # logging

    configuration['logging']['logfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}_simulator.log')

    configuration['logging']['datfile'] = join(configuration['simulator']['output'], f'job_{node_number+1}_simulator.dat')

    configuration['logging']['logfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.log')

    configuration['logging']['datfile'] = join(configuration[mode]['output'], f'job_{node_number+1}_{mode}.dat')

    # write new configuration

    with open(jobname_conf, 'w+') as f:

        dump(configuration, f, default_flow_style=False)



def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode='simulator'):

def make_sh(jobname, slurmconf, jobname_conf, jobname_sh, jobname_log, mode):

    # write sbatch

    with open(jobname_sh, 'w+') as f:

        f.write("#!/bin/bash")
@@ -49,15 +49,15 @@ def make_sbatch(jobname, configuration, node_number, mode): 
    jobname_log = join(output, f"{jobname}_{mode}.slurm")

    jobname_conf = join(output, f"{jobname}_{mode}.yml")

    make_configuration(jobname_conf, configuration, node_number, mode=mode)

    make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log)

    make_sh(jobname, configuration['slurm'], jobname_conf, jobname_sh, jobname_log, mode)

    system(f"sbatch {jobname_sh}")



def slurm_submission(configuration_file, nodes, mode):

    configuration = load_yaml_conf(configuration_file)

    log = set_logger(get_log_level(configuration['logging']['level']))

    # create output dir

    log.info(f"Creating {configuration['simulator']['output']}")

    makedirs(configuration['simulator']['output'], exist_ok=True)

    log.info(f"Creating {configuration[mode]['output']}")

    makedirs(configuration[mode]['output'], exist_ok=True)

    # sbatch jobs per each nodes

    configuration['slurm']['nodes'] = nodes

    for node_number in range(configuration['slurm']['nodes']):

astrort/simulator/base_mapper.py

+3 −3
Original line number Diff line number Diff line
@@ -12,13 +12,13 @@ from os import makedirs 
from os.path import join

from astrort.utils.wrap import load_yaml_conf, write_mapping_info, execute_mapper_no_visibility

from astrort.utils.utils import get_all_seeds

from astrort.configure.logging import set_logger, get_log_level

from astrort.configure.logging import set_logger, get_log_level, get_logfile

from astrort.configure.slurmjobs import slurm_submission



def base_mapper(configuration_file, seeds=None):

    clock = time()

    configuration = load_yaml_conf(configuration_file)

    logfile = join(configuration['mapper']['output'], 'mapper_' + configuration['logging']['logfile'])

    logfile = get_logfile(configuration, mode='mapper')

    datfile = logfile.replace('.log', '.dat')

    # set logger

    log = set_logger(get_log_level(configuration['logging']['level']), logfile)
@@ -48,7 +48,7 @@ def base_mapper(configuration_file, seeds=None): 


def main(configuration, nodes):

    if nodes == 0:

        base_mapper()(configuration)

        base_mapper(configuration)

    else:

        slurm_submission(configuration, nodes, mode='mapper')

astrort/simulator/base_simulator.py

+3 −5
Original line number Diff line number Diff line
@@ -8,23 +8,21 @@ 


import argparse

from time import time

from os import makedirs

from os.path import join

from rtasci.lib.RTACtoolsSimulation import RTACtoolsSimulation

from astrort.utils.wrap import load_yaml_conf, configure_simulator_no_visibility, write_simulation_info, set_pointing

from astrort.configure.logging import set_logger, get_log_level

from astrort.configure.logging import set_logger, get_log_level, get_logfile

from astrort.configure.slurmjobs import slurm_submission



def base_simulator(configuration_file):

    clock = time()

    configuration = load_yaml_conf(configuration_file)

    logfile = join(configuration['simulator']['output'], 'simulator_' + configuration['logging']['logfile'])

    logfile = get_logfile(configuration, mode='simulator')

    datfile = logfile.replace('.log', '.dat')

    log = set_logger(get_log_level(configuration['logging']['level']), logfile)

    log.info(f"Simulator configured, took {time() - clock} s")

    # create output dir

    log.info(f"Output folder: {configuration['simulator']['output']}")

    makedirs(configuration['simulator']['output'], exist_ok=True)

    #makedirs(configuration['simulator']['output'], exist_ok=True)

    # start simulations

    log.info(f"\n {'-'*17} \n| START SIMULATOR | \n {'-'*17} \n")

    for i in range(configuration['simulator']['samples']):

astrort/testing/test_configure/test_logging.py

+12 −2
Original line number Diff line number Diff line
@@ -9,7 +9,8 @@ 
import pytest

import logging

from os.path import join, isfile

from astrort.configure.logging import get_log_level, set_logger

from astrort.configure.logging import *

from astrort.utils.wrap import load_yaml_conf



def test_set_logger():

    assert get_log_level('DEBUG') == logging.DEBUG
@@ -19,8 +20,17 @@ def test_set_logger(): 
    assert get_log_level('CRITICAL') == logging.CRITICAL



@pytest.mark.test_tmp_folder

@pytest.mark.test_conf_file

def test_set_logger(test_tmp_folder):

    log = set_logger(logging.CRITICAL, join(test_tmp_folder, 'test_set_logger.log'))

    log.debug('TEST')

    assert isfile(join(test_tmp_folder, 'test_set_logger.log')) is True



@pytest.mark.test_conf_file

@pytest.mark.parametrize('mode', ['simulator', 'mapper'])

def test_get_logfile(test_conf_file, mode):

    conf = load_yaml_conf(test_conf_file)



    logfile = get_logfile(conf, mode)

    assert mode in logfile