Loading etl/mdl/mdl_execute_radmc_command.py +74 −45 Original line number Diff line number Diff line Loading @@ -85,50 +85,19 @@ def get_command_options(config: dict) -> set: return set(options_list) def main(grid_tarfile: str, override_config: Union[dict, None] = None) -> str: # This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files # The reason for splitting the main input file from the rest is that some parameters can be changed # independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset executed_on = datetime.now() config_stg = load_config_file(os.path.join('stg', 'config', 'config.yml'), override_config=override_config['grid_lines']) config_lines = config_stg['lines'] config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'), override_config=override_config['model']) write_radmc_main_input_file(config_mdl=config_mdl, config_lines=config_lines, path=os.path.join('mdl', 'radmc_files')) assert check_config(config=config_mdl['radmc_observation']) with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile: outfile.write('cd mdl/radmc_files\n') options_set = get_command_options(config_mdl) radmc_command = f'radmc3d image {" ".join(options_set)}' outfile.write(radmc_command) engine = get_pg_engine(logger=logger) os.chdir(os.path.join('mdl', 'radmc_files')) os.system(radmc_command) os.chdir(os.path.join('..', '..')) config_full = config_mdl.copy() config_full.update(config_stg) cube_filename = f'{compute_unique_hash_filename(config=config_full)}.fits' save_cube_as_fits(cube_out_name=cube_filename, cube_out_path=os.path.join('prs', 'fits', 'cubes')) populate_line_table(config_lines=config_lines, engine=engine, executed_on=executed_on, grid_tarfile=grid_tarfile) populate_species_and_partner_table(config_lines, engine, executed_on, grid_tarfile) populate_model_table(config_mdl, grid_tarfile, cube_filename, engine, executed_on) return cube_filename def populate_model_table(config_mdl, grid_tarfile, cube_filename, engine, executed_on): def populate_model_table(config_mdl: dict, grid_tarfile: str, cube_filename: str, engine: sqlalchemy.engine, executed_on: datetime.timestamp): """ Populate the model_parameters table in the DB :param config_mdl: the model configuration :param grid_tarfile: the name of the grid tarfile :param cube_filename: the name of the fits cube :param engine: the SQLAlchemy engine :param executed_on: the timestamp of execution """ model_pars_dict = { 'zipped_grid_name': grid_tarfile, 'fits_cube_name': cube_filename, Loading @@ -153,7 +122,17 @@ def populate_model_table(config_mdl, grid_tarfile, cube_filename, engine, execut ) def populate_species_and_partner_table(config_lines, engine, executed_on, grid_tarfile): def populate_species_and_partner_table(config_lines: dict, engine: sqlalchemy.engine, executed_on: datetime.timestamp, grid_tarfile: str): """ Populate the species_and_partners table in the DB :param config_lines: the line configuration :param engine: the SQLAlchemy engine :param executed_on: the timestamp of execution :param grid_tarfile: the name of the grid tarfile """ for (species, collision_partner) in product(config_lines['species_to_include'], config_lines['collision_partners']): species_partner_dict = { 'zipped_grid_name': f'{grid_tarfile}', Loading Loading @@ -197,5 +176,55 @@ def populate_line_table(config_lines: dict, ) def main(grid_tarfile: str, override_config: Union[dict, None] = None) -> str: # This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files # The reason for splitting the main input file from the rest is that some parameters can be changed # independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset executed_on = datetime.now() config_stg = load_config_file(os.path.join('stg', 'config', 'config.yml'), override_config=override_config['grid_lines']) config_lines = config_stg['lines'] config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'), override_config=override_config['model']) write_radmc_main_input_file(config_mdl=config_mdl, config_lines=config_lines, path=os.path.join('mdl', 'radmc_files')) assert check_config(config=config_mdl['radmc_observation']) with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile: outfile.write('cd mdl/radmc_files\n') options_set = get_command_options(config_mdl) radmc_command = f'radmc3d image {" ".join(options_set)}' outfile.write(radmc_command) engine = get_pg_engine(logger=logger) os.chdir(os.path.join('mdl', 'radmc_files')) os.system(radmc_command) os.chdir(os.path.join('..', '..')) config_full = config_mdl.copy() config_full.update(config_stg) cube_filename = f'{compute_unique_hash_filename(config=config_full)}.fits' save_cube_as_fits(cube_out_name=cube_filename, cube_out_path=os.path.join('prs', 'fits', 'cubes')) populate_line_table(config_lines=config_lines, engine=engine, executed_on=executed_on, grid_tarfile=grid_tarfile) populate_species_and_partner_table(config_lines=config_lines, engine=engine, executed_on=executed_on, grid_tarfile=grid_tarfile) populate_model_table(config_mdl=config_mdl, grid_tarfile=grid_tarfile, cube_filename=cube_filename, engine=engine, executed_on=executed_on) return cube_filename if __name__ == '__main__': main(grid_tarfile='test.tgz') etl/prs/prs_inspect_results.py +9 −21 Original line number Diff line number Diff line Loading @@ -19,32 +19,20 @@ from assets.commons import (load_config_file, logger = setup_logger(name='PRS - INSPECT') # TODO: connect to DB to retrieve filename, given parameters or recompute it if easier def get_fitsfile_name(dust_temperature: float, gas_density: float, lines: Union[list, tuple]) -> str: return f'ratio_td_{str(int(dust_temperature))}_nh2_{str(round(gas_density, 1))}_lines_{"-".join(lines)}.fits' def get_fitsfile_name_from_db( dust_temperature: float, gas_density: float, lines: Union[list, tuple], session: Session) -> str: results = session.query(GridPars, RatioMaps).join(ModelPars).join(MomentZeroMaps).join(RatioMaps, or_(RatioMaps.mom_zero_map_1, RatioMaps.mom_zero_map_2)).filter( and_(GridPars.dust_temperature == dust_temperature, GridPars.central_density == gas_density, or_(ModelPars.iline.in_(lines)))).all() assert len(results) == 1 return results[0][1].ratio_map_name def get_aggregated_ratio_from_db( dust_temperature: float, gas_density: float, lines: Union[list, tuple], session: Session) -> float: """ Get the aggregated ratio from the database, according to the aggregation function specified in the pre config file; this function works for a homogeneous model. For more complex models the query must be revised. :param dust_temperature: the dust temperature of the model :param gas_density: the gas density of the model :param lines: the lines used to compute the ratio :param session: the SQLAlchemy session to use :return: the aggregated ratio """ results = session.query(GridPars, RatioMaps).join(ModelPars).join(MomentZeroMaps).join(RatioMaps, or_(RatioMaps.mom_zero_map_1, RatioMaps.mom_zero_map_2)).filter( Loading Loading
etl/mdl/mdl_execute_radmc_command.py +74 −45 Original line number Diff line number Diff line Loading @@ -85,50 +85,19 @@ def get_command_options(config: dict) -> set: return set(options_list) def main(grid_tarfile: str, override_config: Union[dict, None] = None) -> str: # This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files # The reason for splitting the main input file from the rest is that some parameters can be changed # independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset executed_on = datetime.now() config_stg = load_config_file(os.path.join('stg', 'config', 'config.yml'), override_config=override_config['grid_lines']) config_lines = config_stg['lines'] config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'), override_config=override_config['model']) write_radmc_main_input_file(config_mdl=config_mdl, config_lines=config_lines, path=os.path.join('mdl', 'radmc_files')) assert check_config(config=config_mdl['radmc_observation']) with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile: outfile.write('cd mdl/radmc_files\n') options_set = get_command_options(config_mdl) radmc_command = f'radmc3d image {" ".join(options_set)}' outfile.write(radmc_command) engine = get_pg_engine(logger=logger) os.chdir(os.path.join('mdl', 'radmc_files')) os.system(radmc_command) os.chdir(os.path.join('..', '..')) config_full = config_mdl.copy() config_full.update(config_stg) cube_filename = f'{compute_unique_hash_filename(config=config_full)}.fits' save_cube_as_fits(cube_out_name=cube_filename, cube_out_path=os.path.join('prs', 'fits', 'cubes')) populate_line_table(config_lines=config_lines, engine=engine, executed_on=executed_on, grid_tarfile=grid_tarfile) populate_species_and_partner_table(config_lines, engine, executed_on, grid_tarfile) populate_model_table(config_mdl, grid_tarfile, cube_filename, engine, executed_on) return cube_filename def populate_model_table(config_mdl, grid_tarfile, cube_filename, engine, executed_on): def populate_model_table(config_mdl: dict, grid_tarfile: str, cube_filename: str, engine: sqlalchemy.engine, executed_on: datetime.timestamp): """ Populate the model_parameters table in the DB :param config_mdl: the model configuration :param grid_tarfile: the name of the grid tarfile :param cube_filename: the name of the fits cube :param engine: the SQLAlchemy engine :param executed_on: the timestamp of execution """ model_pars_dict = { 'zipped_grid_name': grid_tarfile, 'fits_cube_name': cube_filename, Loading @@ -153,7 +122,17 @@ def populate_model_table(config_mdl, grid_tarfile, cube_filename, engine, execut ) def populate_species_and_partner_table(config_lines, engine, executed_on, grid_tarfile): def populate_species_and_partner_table(config_lines: dict, engine: sqlalchemy.engine, executed_on: datetime.timestamp, grid_tarfile: str): """ Populate the species_and_partners table in the DB :param config_lines: the line configuration :param engine: the SQLAlchemy engine :param executed_on: the timestamp of execution :param grid_tarfile: the name of the grid tarfile """ for (species, collision_partner) in product(config_lines['species_to_include'], config_lines['collision_partners']): species_partner_dict = { 'zipped_grid_name': f'{grid_tarfile}', Loading Loading @@ -197,5 +176,55 @@ def populate_line_table(config_lines: dict, ) def main(grid_tarfile: str, override_config: Union[dict, None] = None) -> str: # This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files # The reason for splitting the main input file from the rest is that some parameters can be changed # independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset executed_on = datetime.now() config_stg = load_config_file(os.path.join('stg', 'config', 'config.yml'), override_config=override_config['grid_lines']) config_lines = config_stg['lines'] config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'), override_config=override_config['model']) write_radmc_main_input_file(config_mdl=config_mdl, config_lines=config_lines, path=os.path.join('mdl', 'radmc_files')) assert check_config(config=config_mdl['radmc_observation']) with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile: outfile.write('cd mdl/radmc_files\n') options_set = get_command_options(config_mdl) radmc_command = f'radmc3d image {" ".join(options_set)}' outfile.write(radmc_command) engine = get_pg_engine(logger=logger) os.chdir(os.path.join('mdl', 'radmc_files')) os.system(radmc_command) os.chdir(os.path.join('..', '..')) config_full = config_mdl.copy() config_full.update(config_stg) cube_filename = f'{compute_unique_hash_filename(config=config_full)}.fits' save_cube_as_fits(cube_out_name=cube_filename, cube_out_path=os.path.join('prs', 'fits', 'cubes')) populate_line_table(config_lines=config_lines, engine=engine, executed_on=executed_on, grid_tarfile=grid_tarfile) populate_species_and_partner_table(config_lines=config_lines, engine=engine, executed_on=executed_on, grid_tarfile=grid_tarfile) populate_model_table(config_mdl=config_mdl, grid_tarfile=grid_tarfile, cube_filename=cube_filename, engine=engine, executed_on=executed_on) return cube_filename if __name__ == '__main__': main(grid_tarfile='test.tgz')
etl/prs/prs_inspect_results.py +9 −21 Original line number Diff line number Diff line Loading @@ -19,32 +19,20 @@ from assets.commons import (load_config_file, logger = setup_logger(name='PRS - INSPECT') # TODO: connect to DB to retrieve filename, given parameters or recompute it if easier def get_fitsfile_name(dust_temperature: float, gas_density: float, lines: Union[list, tuple]) -> str: return f'ratio_td_{str(int(dust_temperature))}_nh2_{str(round(gas_density, 1))}_lines_{"-".join(lines)}.fits' def get_fitsfile_name_from_db( dust_temperature: float, gas_density: float, lines: Union[list, tuple], session: Session) -> str: results = session.query(GridPars, RatioMaps).join(ModelPars).join(MomentZeroMaps).join(RatioMaps, or_(RatioMaps.mom_zero_map_1, RatioMaps.mom_zero_map_2)).filter( and_(GridPars.dust_temperature == dust_temperature, GridPars.central_density == gas_density, or_(ModelPars.iline.in_(lines)))).all() assert len(results) == 1 return results[0][1].ratio_map_name def get_aggregated_ratio_from_db( dust_temperature: float, gas_density: float, lines: Union[list, tuple], session: Session) -> float: """ Get the aggregated ratio from the database, according to the aggregation function specified in the pre config file; this function works for a homogeneous model. For more complex models the query must be revised. :param dust_temperature: the dust temperature of the model :param gas_density: the gas density of the model :param lines: the lines used to compute the ratio :param session: the SQLAlchemy session to use :return: the aggregated ratio """ results = session.query(GridPars, RatioMaps).join(ModelPars).join(MomentZeroMaps).join(RatioMaps, or_(RatioMaps.mom_zero_map_1, RatioMaps.mom_zero_map_2)).filter( Loading
