Changed postgres minor version; parallelized main. (a5eaef66) · Commits · Andrea Giannetti / swiss_army_knife_stable

etl/assets/commons/init.py

+7 −4

Original line number	Diff line number	Diff line
		@@ -145,7 +145,7 @@ def compute_power_law_radial_profile(
		:return: the distance matrix
		"""
		_value_at_reference = validate_parameter(value_at_reference, default=central_value)
		_distance_matrix = np.where(distance_matrix == 0, 1,
		_distance_matrix = np.where(distance_matrix == 0, np.nanmin(distance_matrix),
		distance_matrix) if fill_reference_pixel is True else distance_matrix
		try:
		_distance_reference = distance_reference.to(u.cm).value
		@@ -279,18 +279,21 @@ def get_physical_px_size(grid_metadata: dict) -> List[u.Quantity]:


		def get_moldata(species_names: list,
		logger: logging.Logger):
		logger: logging.Logger,
		path: Union[str, None] = None):
		"""
		Downloads the molecular data from the Leiden molecular database; check whether the molecule in mapped in
		leiden_url_mapping
		:param species_names: the names of the species for which to download data
		:param logger: logger for output printing
		:param path: path where the files must be saved
		"""
		_path = validate_parameter(path, default=os.path.join('mdl', 'radmc_files'))
		for species in species_names:
		if not os.path.isfile(os.path.join('mdl', 'radmc_files', f'molecule_{species}.inp')):
		if not os.path.isfile(os.path.join(_path, f'molecule_{species}.inp')):
		logger.info(f'Downloading file molecule_{species}.inp...')
		data = urllib.request.urlopen(leiden_url_mapping[species]).read().decode()
		with open(os.path.join('mdl', 'radmc_files', f'molecule_{species}.inp'), 'w') as outfile:
		with open(os.path.join(_path, f'molecule_{species}.inp'), 'w') as outfile:
		outfile.writelines(data)
		else:
		logger.info(f'File molecule_{species}.inp found, skipping download...')

etl/main.py

+56 −22

Original line number	Diff line number	Diff line
		import os
		import uuid
		from itertools import product
		from multiprocessing import Pool
		from assets.commons import (load_config_file,
		parse_grid_overrides)
		from stg.stg_radmc_input_generator import main as stg_main
		@@ -8,6 +10,32 @@ from prs.prs_compute_integrated_fluxes_and_ratios import main as prs_main
		from prs.prs_inspect_results import main as prs_inspection_main


		def compute_grid(tdust, nh2, lines, density_keyword):
		scratch_dir = os.path.join('mdl', 'scratches', str(uuid.uuid4()))
		overrides = {
		'grid': {
		'dust_temperature': tdust,
		density_keyword: nh2,
		}
		}
		tarname = stg_main(override_config=overrides,
		path_radmc_files=scratch_dir)
		cube_fits = []
		for line in lines:
		mdl_overrides = {
		'grid_lines': overrides,
		'model': {
		'radmc_observation': {
		'iline': line
		}
		}
		}
		cube_fits.append(execute_radmc_script(grid_tarfile=tarname,
		override_config=mdl_overrides,
		radmc_input_path=scratch_dir))
		prs_main(cube_fits_list=cube_fits)


		def build_model_grid():
		stg_config = load_config_file(os.path.join('stg', 'config', 'config.yml'))
		pl_density_index = float(stg_config['grid']['density_powerlaw_idx'])
		@@ -26,28 +54,34 @@ def build_model_grid():

		density_keyword = 'central_density' if _model_type == 'homogeneous' else 'density_at_reference'

		for (tdust, nh2) in product(dust_temperatures, densities):
		overrides = {
		'grid': {
		'dust_temperature': tdust,
		density_keyword: nh2,
		}
		}
		grid_tarfiles.append(stg_main(override_config=overrides))
		cube_fits = []
		ratio_fits = []
		for line in lines:
		mdl_overrides = {
		'grid_lines': overrides,
		'model': {
		'radmc_observation': {
		'iline': line
		}
		}
		}
		cube_fits.append(execute_radmc_script(grid_tarfile=grid_tarfiles[-1],
		override_config=mdl_overrides))
		ratio_fits.append(prs_main(cube_fits_list=cube_fits))
		# scratch_dir = os.path.join('mdl', 'scratches', str(uuid.uuid4()))
		# for (tdust, nh2) in product(dust_temperatures, densities):
		# overrides = {
		# 'grid': {
		# 'dust_temperature': tdust,
		# density_keyword: nh2,
		# }
		# }
		# grid_tarfiles.append(stg_main(override_config=overrides,
		# path_radmc_files=scratch_dir))
		# cube_fits = []
		# ratio_fits = []
		# for line in lines:
		# mdl_overrides = {
		# 'grid_lines': overrides,
		# 'model': {
		# 'radmc_observation': {
		# 'iline': line
		# }
		# }
		# }
		# cube_fits.append(execute_radmc_script(grid_tarfile=grid_tarfiles[-1],
		# override_config=mdl_overrides,
		# radmc_input_path=scratch_dir))
		# ratio_fits.append(prs_main(cube_fits_list=cube_fits))
		parallel_args = product(dust_temperatures, densities, [lines], [density_keyword])
		with Pool(4) as pool:
		pool.starmap(compute_grid, parallel_args)
		prs_inspection_main()

etl/mdl/mdl_execute_radmc_command.py

+24 −10

Original line number	Diff line number	Diff line
		@@ -16,7 +16,8 @@ from assets.commons import (load_config_file,
		get_pg_engine,
		setup_logger,
		compute_unique_hash_filename,
		get_value_if_specified)
		get_value_if_specified,
		cleanup_directory)
		from assets.constants import (radmc_options_mapping,
		radmc_lines_mode_mapping)

		@@ -42,15 +43,18 @@ def write_radmc_main_input_file(config_mdl: dict,


		def save_cube_as_fits(cube_out_name: Union[str, None] = None,
		cube_out_path: Union[str, None] = None):
		cube_out_path: Union[str, None] = None,
		path_radmc_files: Union[str, None] = None):
		"""
		Convert the RADMC output to fits
		:param cube_out_name: outfile name
		:param cube_out_path: outfile path
		:param path_radmc_files: path where the radmc input/output files are stored
		"""
		_cube_out_name = validate_parameter(cube_out_name, default='test_cube.fits')
		_cube_out_path = validate_parameter(cube_out_path, default=os.path.join('prs', 'fits', 'cubes'))
		imdata = image.readImage(fname=os.path.join('mdl', 'radmc_files', 'image.out'))
		_path_radmc_files = validate_parameter(path_radmc_files, default=os.path.join('mdl', 'radmc_files'))
		imdata = image.readImage(fname=os.path.join(_path_radmc_files, 'image.out'))
		output_name = os.path.join(_cube_out_path, _cube_out_name)
		if os.path.isfile(output_name):
		os.remove(output_name)
		@@ -177,12 +181,14 @@ def populate_line_table(config_lines: dict,


		def main(grid_tarfile: str,
		override_config: Union[dict, None] = None) -> str:
		override_config: Union[dict, None] = None,
		radmc_input_path: Union[str, None] = None) -> str:
		# This is necessary, because the lines_mode is needed both in the lines.inp and radmc3d.inp files
		# The reason for splitting the main input file from the rest is that some parameters can be changed
		# independently of the grid for the modeling. The mdl hash should depend on all the mdl parameters, not a subset
		_override_config = validate_parameter(override_config, default={'grid_lines': {}, 'model': {}})
		executed_on = datetime.now()
		_radmc_input_path = validate_parameter(radmc_input_path, default=os.path.join('mdl', 'radmc_files'))
		config_stg = load_config_file(os.path.join('stg', 'config', 'config.yml'),
		override_config=_override_config['grid_lines'])
		config_lines = config_stg['lines']
		@@ -190,24 +196,32 @@ def main(grid_tarfile: str,
		override_config=_override_config['model'])
		write_radmc_main_input_file(config_mdl=config_mdl,
		config_lines=config_lines,
		path=os.path.join('mdl', 'radmc_files'))
		path=_radmc_input_path)
		assert check_config(config=config_mdl['radmc_observation'])
		with open(os.path.join('mdl', 'radmc3d_postprocessing.sh'), 'w') as outfile:
		outfile.write('cd mdl/radmc_files\n')
		outfile.write(f'cd {_radmc_input_path}\n')
		options_set = get_command_options(config_mdl)
		radmc_command = f'radmc3d image setthreads 4 {" ".join(options_set)}'
		outfile.write(radmc_command)

		engine = get_pg_engine(logger=logger)
		os.chdir(os.path.join('mdl', 'radmc_files'))

		# Execute radmc
		execution_dir = os.getcwd()
		os.chdir(_radmc_input_path)
		os.system(radmc_command)
		os.chdir(os.path.join('..', '..'))
		os.chdir(execution_dir)
		if radmc_input_path is not None:
		cleanup_directory(directory=_radmc_input_path, logger=logger)
		os.rmdir(_radmc_input_path)

		config_full = config_mdl.copy()
		config_full.update(config_stg)
		cube_filename = f'{compute_unique_hash_filename(config=config_full)}.fits'

		save_cube_as_fits(cube_out_name=cube_filename,
		cube_out_path=os.path.join('prs', 'fits', 'cubes'))
		cube_out_path=os.path.join('prs', 'fits', 'cubes'),
		path_radmc_files=radmc_input_path)

		populate_line_table(config_lines=config_lines,
		engine=engine,
		@@ -228,4 +242,4 @@ def main(grid_tarfile: str,


		if __name__ == '__main__':
		main(grid_tarfile='test.tgz')
		main(grid_tarfile='459295aa894dffa8c521e606d14dbb6927638a2c.tgz')

etl/stg/stg_radmc_input_generator.py

+25 −13

Original line number	Diff line number	Diff line
		@@ -79,11 +79,14 @@ def write_radmc_lines_input(line_config: dict,
		outfile.write(f'{coll_partner}\n')


		def copy_additional_files(files_to_transfer: Union[None, List[str]] = None):
		def copy_additional_files(files_to_transfer: Union[None, List[str]] = None,
		dst_path: Union[str, None] = None):
		_dst_path = validate_parameter(dst_path, default=os.path.join('mdl', 'radmc_files'))
		_files_to_transfer = validate_parameter(files_to_transfer, default=['dustkappa_silicate.inp',
		'dustopac.inp'])
		_path = os.path.join('mdl', 'data')
		_dst_path = os.path.join('mdl', 'radmc_files')
		if not os.path.isdir(_dst_path):
		os.mkdir(_dst_path)
		for filename in _files_to_transfer:
		shutil.copy2(os.path.join(_path, filename), _dst_path)

		@@ -114,7 +117,7 @@ def get_solid_body_rotation_y(grid_metadata):
		return velocity_x, velocity_z


		def get_profiles(grid_metadata):
		def get_profiles(grid_metadata: dict) -> dict:
		try:
		density_ref = {
		'value_at_reference': (
		@@ -195,7 +198,10 @@ def get_profiles(grid_metadata):
		return profiles


		def write_grid_input_files(profiles, grid_metadata):
		def write_grid_input_files(profiles: dict,
		grid_metadata: dict,
		path: Union[str, None] = None):
		_path = validate_parameter(path, default=os.path.join('mdl', 'radmc_files'))
		n_wavelengths = 100
		start_wavelength = 5
		end_wavelength = 1300
		@@ -228,18 +234,20 @@ def write_grid_input_files(profiles, grid_metadata):
		for filename in quantity_mapping:
		write_radmc_input(filename=filename,
		quantity=quantity_mapping[filename]['quantity'],
		path=os.path.join('mdl', 'radmc_files'),
		path=_path,
		grid_metadata=grid_metadata)


		def write_molecular_number_density_profiles(profiles: dict,
		line_config: dict,
		grid_metadata: dict):
		grid_metadata: dict,
		path: Union[str, None] = None):
		_path = validate_parameter(path, default=os.path.join('mdl', 'radmc_files'))
		for species in line_config['species_to_include'] + line_config['collision_partners']:
		write_radmc_input(filename=f'numberdens_{species}.inp',
		quantity=profiles['gas_number_density'] * float(
		line_config['molecular_abundances'][species]),
		path=os.path.join('mdl', 'radmc_files'),
		path=_path,
		grid_metadata=grid_metadata)


		@@ -349,28 +357,32 @@ def populate_grid_table(config: dict,
		return output_filename


		def main(override_config: Union[dict, None] = None) -> str:
		def main(override_config: Union[dict, None] = None,
		path_radmc_files: Union[str, None] = None) -> str:
		config = load_config_file(os.path.join('stg', 'config', 'config.yml'),
		override_config=override_config)
		engine = get_pg_engine(logger=logger)

		input_files_dir = os.path.join('mdl', 'radmc_files')
		input_files_dir = validate_parameter(path_radmc_files, default=os.path.join('mdl', 'radmc_files'))
		cleanup_directory(directory=input_files_dir,
		logger=logger)
		copy_additional_files()
		copy_additional_files(dst_path=input_files_dir)

		grid_metadata = extract_grid_metadata(config=config)
		profiles = get_profiles(grid_metadata=grid_metadata)
		write_grid_input_files(grid_metadata=grid_metadata,
		profiles=profiles)
		profiles=profiles,
		path=input_files_dir)
		get_moldata(species_names=config['lines']['species_to_include'],
		logger=logger)
		logger=logger,
		path=input_files_dir)
		write_radmc_lines_input(line_config=config['lines'],
		path=input_files_dir,
		logger=logger)
		write_molecular_number_density_profiles(profiles=profiles,
		grid_metadata=grid_metadata,
		line_config=config['lines'])
		line_config=config['lines'],
		path=input_files_dir)
		tgz_filename = populate_grid_table(config=config, engine=engine, grid_metadata=grid_metadata)
		make_tarfile(output_filename=tgz_filename,
		source_dir=input_files_dir,