Persisted mdl parameters to DB; refactored the MDL layer. (516b7369) · Commits · Andrea Giannetti / swiss_army_knife_stable

documentation/documentation.md

+12 −12

Original line number	Diff line number	Diff line
		@@ -32,12 +32,12 @@ The SAK non-LTE, toy model pipeline uses three main layers:
		The `etl/stg/config/config.yml` file contains the default values of the parameters used to prepare the RADMC files
		for the postprocessing. All of them are described in detail in the following.

		2. Model layer: The model layer (`etl/mdl`) takes care of preparing the RADMC command according to the configuration
		in the `etl/mdl/config/config.yml` file. This is done by the `mdl_prepare_radmc_command.py` script, that
		creates `radmc3d_postprocessing.sh`.
		2. Model layer: The model layer (`etl/mdl`) takes care of preparing and executing the RADMC command according to the
		configuration in the `etl/mdl/config/config.yml` file. This is done by the `mdl_execute_radmc_command.py` script,
		that also creates `radmc3d_postprocessing.sh`.

		The results can then be converted to fits cubes via the `mdl_save_results_as_fits.py` script. The cubes are saved by
		default into `prs/fits/cubes`, for later processing.
		The results are then converted to fits cubes, which are saved by default into `prs/fits/cubes`, for later
		processing.

		3. Presentation layer: In the presentation layer (`etl/prs`) moment-0 maps and line-ratio maps are computed
		executing the `prs_compute_integrated_fluxes_and_ratios.py` script. At the moment, the integration limits cannot be
		@@ -60,7 +60,7 @@ their meaning.

		#### The staging configuration file (`etl/stg/config/config.yml`)

		The staging config file has three main categories:
		The staging config file has two main categories:

		* grid:
		* grid_type: regular or spherical
		@@ -98,7 +98,11 @@ The staging config file has three main categories:
		* collision_partners: the list of collision partners to be used; it must appear in the correct order as in the
		molecule_{molname}.inp file of the molecule to be simulated, e.g. ['p-h2']

		* radmc:
		#### The model configuration file (`etl/mdl/config/config.yml`)

		The model configuration file has two categories:

		* radmc_postprocessing:
		* nphotons: the number of photons to use for the postprocessing
		* scattering_mode_max: override the scattering settings in the dust opacity files; 0 excludes scattering, 1 treats
		it in an isotropic way (if defined), 2 includes anisotropic scattering (if defined)
		@@ -106,11 +110,7 @@ The staging config file has three main categories:
		* tgas_eq_tdust: whether the gas temperature is equal to the dust temperature (if not, it must be specified or
		computed separately!)

		#### The model configuration file (`etl/mdl/config/config.yml`)

		The model configuration file has only the "radmc" category:

		* radmc:
		* radmc_observation:
		* inclination: the inclination wrt. the line-of-sight in degrees
		* position_angle: the position angle wrt. the observer
		* imolspec: the index of the species to be modeled; defaults to the first

etl/assets/commons/init.py

+15 −0

Original line number	Diff line number	Diff line
		@@ -329,6 +329,13 @@ def upsert(table_object: sqla.orm.decl_api.DeclarativeMeta,
		row_dict: dict,
		conflict_keys: List[sqla.Column],
		engine: sqla.engine):
		"""
		Upsert the row into the specified table, according to the indicated conflict columns
		:param table_object: the table into which the row must be inserted
		:param row_dict: the dictionary representing the row to insert
		:param conflict_keys: the conflict columns list, representing the primary key of the table
		:param engine: the SQLAlchemy engine to use
		"""
		statement = insert(table_object).values(row_dict)
		statement = statement.on_conflict_do_update(
		index_elements=conflict_keys, set_=row_dict)
		@@ -348,3 +355,11 @@ def get_pg_engine(logger: logging.Logger) -> sqlalchemy.engine.Engine:
		url = f"postgresql://{credentials['DB_USER']}:{credentials['DB_PASS']}@{credentials['DB_HOST']}:{credentials['DB_PORT']}/{credentials['DB_NAME']}"
		engine = sqlalchemy.create_engine(url)
		return engine


		def get_value_if_specified(parameters_dict: dict,
		key: str):
		try:
		return parameters_dict[key]
		except KeyError:
		return None

etl/main.py

+9 −12

Original line number	Diff line number	Diff line
		@@ -4,8 +4,7 @@ from itertools import product
		from assets.commons import (load_config_file,
		parse_grid_overrides)
		from stg.stg_radmc_input_generator import main as stg_main
		from mdl.mdl_prepare_radmc_command import main as create_radmc_script
		from mdl.mdl_save_results_as_fits import main as save_results
		from mdl.mdl_execute_radmc_command import main as execute_radmc_script
		from prs.prs_compute_integrated_fluxes_and_ratios import main as prs_main


		@@ -19,27 +18,25 @@ def build_model_grid():
		config=config)
		lines = config['overrides']['lines_to_process']

		for (tdust, nH2) in product(dust_temperatures, central_densities):
		for (tdust, nh2) in product(dust_temperatures, central_densities):
		overrides = {
		'grid': {
		'dust_temperature': tdust,
		'central_density': nH2,
		'central_density': nh2,
		}
		}
		stg_main(override_config=overrides)
		tarfile_filename = stg_main(override_config=overrides)
		for line in lines:
		line_overrides = {
		'radmc': {
		'iline': line
		}
		}
		create_radmc_script(override_config=line_overrides)
		os.system('chmod u+x ./mdl/radmc3d_postprocessing.sh')
		os.system('./mdl/radmc3d_postprocessing.sh')
		save_results(cube_out_name=f'td_{tdust}_nh2_{nH2}_line_{line}.fits')
		prs_main(cube_fits1=f'td_{tdust}_nh2_{nH2}_line_{lines[0]}.fits',
		cube_fits2=f'td_{tdust}_nh2_{nH2}_line_{lines[1]}.fits',
		ratio_fits_name=f'ratio_td_{tdust}_nh2_{nH2}_lines_{lines[0]}-{lines[1]}.fits')
		execute_radmc_script(grid_tarfile=tarfile_filename,
		override_config=line_overrides)
		prs_main(cube_fits1=f'td_{tdust}_nh2_{nh2}_line_{lines[0]}.fits',
		cube_fits2=f'td_{tdust}_nh2_{nh2}_line_{lines[1]}.fits',
		ratio_fits_name=f'ratio_td_{tdust}_nh2_{nh2}_lines_{lines[0]}-{lines[1]}.fits')


		if __name__ == '__main__':

etl/mdl/config/config.yml

+1 −1

Original line number	Diff line number	Diff line
		radmc_postprocessing:
		nphotons: 1000000,
		nphotons: 1000000
		scattering_mode_max: 0
		iranfreqmode: 1
		tgas_eq_tdust: 1

etl/mdl/mdl_prepare_radmc_command.py→etl/mdl/mdl_execute_radmc_command.py

+153 −0

Original line number	Diff line number	Diff line
		import os
		import sys
		from itertools import product
		from typing import Union
		from astropy import units as u
		from radmc3dPy import image
		from stg.stg_build_db_structure import (LinePars,
		SpeciesAndPartners,
		ModelPars)
		from assets.commons import (load_config_file,
		validate_parameter)
		validate_parameter,
		upsert,
		get_pg_engine,
		setup_logger,
		compute_unique_hash_filename,
		get_value_if_specified)
		from assets.constants import (radmc_options_mapping,
		radmc_lines_mode_mapping)

		@@ -19,22 +30,15 @@ def write_radmc_main_input_file(config_mdl: dict,
		outfile.write(f'lines_mode = {radmc_lines_mode_mapping[config_lines["lines_mode"]]}\n')


		def main(override_config: Union[dict, None] = None):
		# This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files
		# The reason for splitting the main input file from the rest is that some parameters can be changed
		# independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset
		config_lines = load_config_file(os.path.join('stg', 'config', 'config.yml'),
		override_config=override_config)['lines']
		config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'),
		override_config=override_config)
		write_radmc_main_input_file(config_mdl=config_mdl,
		config_lines=config_lines,
		path=os.path.join('mdl', 'radmc_files'))
		assert check_config(config=config_mdl)
		with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile:
		outfile.write('cd mdl/radmc_files\n')
		options_set = get_command_options(config_mdl)
		outfile.write(f'radmc3d image {" ".join(options_set)}')
		def save_cube_as_fits(cube_out_name: Union[str, None] = None,
		cube_out_path: Union[str, None] = None):
		_cube_out_name = validate_parameter(cube_out_name, default='test_cube.fits')
		_cube_out_path = validate_parameter(cube_out_path, default=os.path.join('prs', 'fits', 'cubes'))
		imdata = image.readImage(fname=os.path.join('mdl', 'radmc_files', 'image.out'))
		output_name = os.path.join(_cube_out_path, _cube_out_name)
		if os.path.isfile(output_name):
		os.remove(output_name)
		imdata.writeFits(fname=output_name)


		def check_config(config: dict) -> bool:
		@@ -65,5 +69,85 @@ def get_command_options(config: dict) -> set:
		return set(options_list)


		def main(grid_tarfile: str,
		override_config: Union[dict, None] = None) -> str:
		# This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files
		# The reason for splitting the main input file from the rest is that some parameters can be changed
		# independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset
		logger = setup_logger(name='MDL')
		config_lines = load_config_file(os.path.join('stg', 'config', 'config.yml'),
		override_config=override_config)['lines']
		config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'),
		override_config=override_config)
		write_radmc_main_input_file(config_mdl=config_mdl,
		config_lines=config_lines,
		path=os.path.join('mdl', 'radmc_files'))
		assert check_config(config=config_mdl['radmc_observation'])
		with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile:
		outfile.write('cd mdl/radmc_files\n')
		options_set = get_command_options(config_mdl)
		radmc_command = f'radmc3d image {" ".join(options_set)}'
		outfile.write(radmc_command)

		engine = get_pg_engine(logger=logger)
		os.chdir(os.path.join('mdl', 'radmc_files'))
		os.system(radmc_command)
		os.chdir(os.path.join('..', '..'))
		cube_filename = f'{compute_unique_hash_filename(config=config_mdl)}.fits'

		save_cube_as_fits(cube_out_name=cube_filename,
		cube_out_path=os.path.join('prs', 'fits', 'cubes'))

		line_pars_dict = {
		'zipped_grid_name': f'{grid_tarfile}',
		'lines_mode': config_lines['lines_mode']
		}
		upsert(
		table_object=LinePars,
		row_dict=line_pars_dict,
		conflict_keys=[LinePars.zipped_grid_name],
		engine=engine
		)

		for (species, collision_partner) in product(config_lines['species_to_include'], config_lines['collision_partners']):
		species_partner_dict = {
		'zipped_grid_name': f'{grid_tarfile}',
		'species_to_include': species,
		'molecular_abundance': config_lines['molecular_abundances'][species],
		'collision_partner': collision_partner,
		'molecular_abundance_collision_partner': config_lines['molecular_abundances'][collision_partner]
		}
		upsert(
		table_object=SpeciesAndPartners,
		row_dict=species_partner_dict,
		conflict_keys=[SpeciesAndPartners.zipped_grid_name,
		SpeciesAndPartners.species_to_include,
		SpeciesAndPartners.collision_partner],
		engine=engine
		)

		model_pars_dict = {
		'fits_cube_name': cube_filename,
		'nphotons': config_mdl['radmc_postprocessing']['nphotons'],
		'scattering_mode_max': int(config_mdl['radmc_postprocessing']['scattering_mode_max']),
		'iranfreqmode': config_mdl['radmc_postprocessing']['iranfreqmode'],
		'tgas_eq_tdust': config_mdl['radmc_postprocessing']['tgas_eq_tdust'],
		'inclination': config_mdl['radmc_observation']['inclination'],
		'position_angle': config_mdl['radmc_observation']['position_angle'],
		'imolspec': get_value_if_specified(config_mdl['radmc_observation'], 'imolspec'),
		'iline': get_value_if_specified(config_mdl['radmc_observation'], 'iline'),
		'width_kms': get_value_if_specified(config_mdl['radmc_observation'], 'width_kms'),
		'nchannels': get_value_if_specified(config_mdl['radmc_observation'], 'nchannels'),
		'npix': get_value_if_specified(config_mdl['radmc_observation'], 'npix'),
		}
		upsert(
		table_object=ModelPars,
		row_dict=model_pars_dict,
		conflict_keys=[ModelPars.fits_cube_name],
		engine=engine
		)
		return cube_filename


		if __name__ == '__main__':
		main()
		main(grid_tarfile='test.tgz')