Merge branch 'refactor_temperature_update' (98c0356a) · Commits · Andrea Giannetti / swiss_army_knife_stable

etl/main.py

+7 −7

Original line number	Diff line number	Diff line
		@@ -16,25 +16,25 @@ from prs.prs_inspect_results import main as prs_inspection_main

		def compute_grid(tdust, nh2, line, density_keyword, dust_temperature_keyword):
		scratch_dir = os.path.join('mdl', 'scratches', str(uuid.uuid4()))
		overrides = {
		stg_overrides = {
		'grid': {
		dust_temperature_keyword: tdust,
		density_keyword: nh2,
		}
		}
		tarname = stg_main(override_config=overrides,
		path_radmc_files=scratch_dir,
		run_id=run_id)
		mdl_overrides = {
		'grid_lines': overrides,
		overrides = {
		'grid_lines': stg_overrides,
		'model': {
		'radmc_observation': {
		'iline': line
		}
		}
		}
		tarname = stg_main(override_config=overrides,
		path_radmc_files=scratch_dir,
		run_id=run_id)
		cube_fits_name = execute_radmc_script(grid_zipfile=tarname,
		override_config=mdl_overrides,
		override_config=overrides,
		radmc_input_path=scratch_dir,
		run_id=run_id)
		return nh2, tdust, line, cube_fits_name

etl/mdl/mdl_execute_radmc_command.py

+1 −61

Original line number	Diff line number	Diff line
		@@ -25,22 +25,6 @@ from assets.constants import (radmc_options_mapping,
		logger = setup_logger(name='MDL')


		def write_radmc_main_input_file(config_mdl: dict,
		config_lines: dict,
		path: Union[None, str] = None):
		"""
		Creates the main input file for RADMC, which is later used for execution
		:param config_mdl: the model configuration
		:param config_lines: te line configuration, from the STG layer
		:param path: the path where to put the input file
		"""
		_path = validate_parameter(path, default='.')
		with open(os.path.join(_path, 'radmc3d.inp'), "w") as outfile:
		for parameter in config_mdl["radmc_postprocessing"]:
		outfile.write(f'{parameter} = {config_mdl["radmc_postprocessing"][parameter]}\n')
		outfile.write(f'lines_mode = {radmc_lines_mode_mapping[config_lines["lines_mode"]]}\n')


		def save_cube_as_fits(cube_out_name: Union[str, None] = None,
		cube_out_path: Union[str, None] = None,
		path_radmc_files: Union[str, None] = None):
		@@ -189,37 +173,10 @@ def populate_line_table(config_lines: dict,
		)


		def populate_stars_table(config_stars: dict,
		engine: sqlalchemy.engine,
		executed_on: datetime.timestamp,
		grid_zipfile: str,
		run_id: str):
		"""
		Populate the stars table in the DB
		:param config_stars: the dictionary containing the stars configurations
		:param engine: the SQLAlchemy engine to use
		:param executed_on: the timestamp of execution, to add to the record
		:param grid_zipfile: the grid tarfile name, to be used as key
		:param run_id: the run unique identifier
		"""
		_config_stars = config_stars.copy()
		_config_stars['zipped_grid_name'] = grid_zipfile
		_config_stars['created_on'] = executed_on
		_config_stars['run_id'] = run_id

		upsert(
		table_object=StarsPars,
		row_dict=_config_stars,
		conflict_keys=[StarsPars.zipped_grid_name, StarsPars.run_id],
		engine=engine
		)


		def main(grid_zipfile: str,
		run_id: str,
		override_config: Union[dict, None] = None,
		radmc_input_path: Union[str, None] = None,
		compute_dust_temperature: bool = True) -> str:
		radmc_input_path: Union[str, None] = None) -> str:
		# This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files
		# The reason for splitting the main input file from the rest is that some parameters can be changed
		# independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset
		@@ -231,9 +188,6 @@ def main(grid_zipfile: str,
		config_lines = config_stg['lines']
		config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'),
		override_config=_override_config['model'])
		write_radmc_main_input_file(config_mdl=config_mdl,
		config_lines=config_lines,
		path=_radmc_input_path)
		assert check_config(config=config_mdl['radmc_observation'])
		with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile:
		outfile.write(f'cd {_radmc_input_path}\n')
		@@ -247,20 +201,6 @@ def main(grid_zipfile: str,
		logger.debug(f'Executing command: {radmc_command}')
		execution_dir = os.getcwd()
		os.chdir(_radmc_input_path)
		if 'stars' in config_stg:
		if compute_dust_temperature is True:
		logger.info('Computing dust temperature distribution using the stars in the configuration file')
		os.system('radmc3d mctherm setthreads 16')
		os.system(f'zip -ur {os.path.join(execution_dir, "stg", "archive", grid_zipfile)} dust_temperature.dat')
		populate_stars_table(config_stars=config_stg['stars'],
		engine=engine,
		executed_on=executed_on,
		grid_zipfile=grid_zipfile,
		run_id=run_id)
		else:
		logger.info('Using cached dust temperature distribution')
		shutil.copy(os.path.join(execution_dir, 'model', 'data', 'dust_temperature.dat'),
		os.path.join('.', 'dust_temperature.dat'))
		os.system(radmc_command)
		os.chdir(execution_dir)
		logger.debug(f'Checking presence of file: {os.path.join(_radmc_input_path, "image.out")}')

etl/stg/stg_radmc_input_generator.py

+63 −9

Original line number	Diff line number	Diff line
		@@ -22,7 +22,7 @@ from assets.commons import (compute_power_law_radial_profile,
		from assets.constants import (mean_molecular_mass,
		radmc_input_headers,
		radmc_lines_mode_mapping)
		from stg.stg_build_db_structure import GridPars, GridFiles
		from stg.stg_build_db_structure import GridPars, GridFiles, StarsPars
		from astropy import units as u
		from astropy import constants as cst
		from astropy.io import fits
		@@ -96,15 +96,20 @@ def copy_additional_files(files_to_transfer: Union[None, List[str]] = None,
		shutil.copy2(os.path.join(_path, filename), _dst_path)


		def write_radmc_main_input_file(config: dict,
		def write_radmc_main_input_file(config_mdl: dict,
		config_lines: dict,
		path: Union[None, str] = None):
		"""
		Creates the main input file for RADMC, which is later used for execution
		:param config_mdl: the model configuration
		:param config_lines: te line configuration, from the STG layer
		:param path: the path where to put the input file
		"""
		_path = validate_parameter(path, default='.')
		with open(os.path.join(_path, 'radmc3d.inp'), "w") as outfile:
		outfile.write(f'nphot = {config["radmc_postprocessing"]["nphotons"]}\n')
		outfile.write(f'scattering_mode_max = {config["radmc_postprocessing"]["scattering_mode_max"]}\n')
		outfile.write(f'iranfreqmode = {config["radmc_postprocessing"]["iranfreqmode"]}\n')
		outfile.write(f'tgas_eq_tdust = {config["radmc_postprocessing"]["tgas_eq_tdust"]}\n')
		outfile.write(f'lines_mode = {radmc_lines_mode_mapping[config["lines"]["lines_mode"]]}\n')
		for parameter in config_mdl["radmc_postprocessing"]:
		outfile.write(f'{parameter} = {config_mdl["radmc_postprocessing"][parameter]}\n')
		outfile.write(f'lines_mode = {radmc_lines_mode_mapping[config_lines["lines_mode"]]}\n')


		def get_solid_body_rotation_y(grid_metadata):
		@@ -401,6 +406,31 @@ def populate_grid_files(quantity: str,
		)


		def populate_stars_table(config_stars: dict,
		engine: sqla_engine,
		grid_zipfile: str,
		run_id: str):
		"""
		Populate the stars table in the DB
		:param config_stars: the dictionary containing the stars configurations
		:param engine: the SQLAlchemy engine to use
		:param executed_on: the timestamp of execution, to add to the record
		:param grid_zipfile: the grid tarfile name, to be used as key
		:param run_id: the run unique identifier
		"""
		_config_stars = config_stars.copy()
		_config_stars['zipped_grid_name'] = grid_zipfile
		_config_stars['created_on'] = datetime.now()
		_config_stars['run_id'] = run_id

		upsert(
		table_object=StarsPars,
		row_dict=_config_stars,
		conflict_keys=[StarsPars.zipped_grid_name, StarsPars.run_id],
		engine=engine
		)


		def write_stellar_input_file(stars_metadata: dict,
		grid_metadata: dict,
		path: str,
		@@ -424,9 +454,14 @@ def write_stellar_input_file(stars_metadata: dict,

		def main(run_id: str,
		override_config: Union[dict, None] = None,
		path_radmc_files: Union[str, None] = None) -> str:
		path_radmc_files: Union[str, None] = None,
		compute_dust_temperature: bool = True) -> str:
		_override_config = validate_parameter(override_config, default={'grid_lines': {}, 'model': {}})
		config = load_config_file(os.path.join('stg', 'config', 'config.yml'),
		override_config=override_config)
		override_config=_override_config['grid_lines'])
		config_lines = config['lines']
		config_mdl = load_config_file(os.path.join('mdl', 'config', 'config.yml'),
		override_config=_override_config['model'])
		engine = get_pg_engine(logger=logger)

		input_files_dir = validate_parameter(path_radmc_files, default=os.path.join('mdl', 'radmc_files'))
		@@ -490,6 +525,25 @@ def main(run_id: str,
		filename=grid_file_name,
		run_id=run_id)

		write_radmc_main_input_file(config_mdl=config_mdl,
		config_lines=config_lines,
		path=input_files_dir)
		execution_dir = os.getcwd()
		os.chdir(input_files_dir)
		if 'stars' in config:
		if compute_dust_temperature is True:
		logger.info('Computing dust temperature distribution using the stars in the configuration file')
		os.system('radmc3d mctherm setthreads 16')
		populate_stars_table(config_stars=config['stars'],
		engine=engine,
		grid_zipfile=zip_filename,
		run_id=run_id)
		else:
		logger.info('Using cached dust temperature distribution')
		shutil.copy(os.path.join(execution_dir, 'model', 'data', 'dust_temperature.dat'),
		os.path.join('.', 'dust_temperature.dat'))
		os.chdir(execution_dir)

		make_archive(output_filename=zip_filename,
		source_dir=input_files_dir,
		archive_path=os.path.join('stg', 'archive'))