Commit ef4a30cc authored by Andrea Giannetti's avatar Andrea Giannetti
Browse files

Changed format of compressed archive to zip in order to update dust... 

Changed format of compressed archive to zip in order to update dust temperature when computed by radmc; added star table and persisting of stellar properties.
parent a7ac0cc5

etl/assets/commons/__init__.py

+8 −5
Original line number Diff line number Diff line
@@ -317,20 +317,23 @@ def compute_unique_hash_filename(config: dict) -> str: 
    return f'{hashed_dict.hexdigest()}'





def make_tarfile(output_filename: str,

def make_archive(output_filename: str,

                 source_dir: str,

                 archive_path: Union[str, None] = None):

    """

    Compresses all files in a directory into a .tgz file

    Compresses all files in a directory into a .zip file

    :param output_filename: the output filename

    :param source_dir: the directory to compress

    :param archive_path: path to store the compressed-grid archives

    """

    _archive_path = validate_parameter(archive_path,

                                       default=os.path.join('stg', 'archive'))

    with tarfile.open(os.path.join(_archive_path, output_filename), "w:gz") as tar:

        tar.add(source_dir,

                arcname=os.path.basename(source_dir))

    filename_root, filename_format = output_filename.rsplit('.', maxsplit=1)

    if os.path.isfile(os.path.join(_archive_path, output_filename)):

        os.remove(os.path.join(_archive_path, output_filename))

    shutil.make_archive(base_name=os.path.join(_archive_path, filename_root),

                        format=filename_format,

                        root_dir=source_dir)





def cleanup_directory(directory: str,

etl/mdl/mdl_execute_radmc_command.py

+48 −15
Original line number Diff line number Diff line
@@ -9,7 +9,8 @@ from datetime import datetime 
from radmc3dPy import image

from stg.stg_build_db_structure import (LinePars,

                                        SpeciesAndPartners,

                                        ModelPars)

                                        ModelPars,

                                        StarsPars)

from assets.commons import (load_config_file,

                            validate_parameter,

                            upsert,
@@ -88,7 +89,7 @@ def get_command_options(config: dict) -> set: 




def populate_model_table(config_mdl: dict,

                         grid_tarfile: str,

                         grid_zipfile: str,

                         cube_filename: str,

                         engine: sqlalchemy.engine,

                         executed_on: datetime.timestamp,
@@ -96,14 +97,14 @@ def populate_model_table(config_mdl: dict, 
    """

    Populate the model_parameters table in the DB

    :param config_mdl: the model configuration

    :param grid_tarfile: the name of the grid tarfile

    :param grid_zipfile: the name of the grid tarfile

    :param cube_filename: the name of the fits cube

    :param engine: the SQLAlchemy engine

    :param executed_on: the timestamp of execution

    :param run_id: the run unique identifier

    """

    model_pars_dict = {

        'zipped_grid_name': grid_tarfile,

        'zipped_grid_name': grid_zipfile,

        'fits_cube_name': cube_filename,

        'nphotons': config_mdl['radmc_postprocessing']['nphot'],

        'scattering_mode_max': int(config_mdl['radmc_postprocessing']['scattering_mode_max']),
@@ -130,19 +131,19 @@ def populate_model_table(config_mdl: dict, 
def populate_species_and_partner_table(config_lines: dict,

                                       engine: sqlalchemy.engine,

                                       executed_on: datetime.timestamp,

                                       grid_tarfile: str,

                                       grid_zipfile: str,

                                       run_id: str):

    """

    Populate the species_and_partners table in the DB

    :param config_lines: the line configuration

    :param engine: the SQLAlchemy engine

    :param executed_on: the timestamp of execution

    :param grid_tarfile: the name of the grid tarfile

    :param grid_zipfile: the name of the grid tarfile

    :param run_id: the run unique identifier

    """

    for (species, collision_partner) in product(config_lines['species_to_include'], config_lines['collision_partners']):

        species_partner_dict = {

            'zipped_grid_name': f'{grid_tarfile}',

            'zipped_grid_name': f'{grid_zipfile}',

            'species_to_include': species,

            'molecular_abundance': config_lines['molecular_abundances'][species],

            'collision_partner': collision_partner,
@@ -164,18 +165,18 @@ def populate_species_and_partner_table(config_lines: dict, 
def populate_line_table(config_lines: dict,

                        engine: sqlalchemy.engine,

                        executed_on: datetime.timestamp,

                        grid_tarfile: str,

                        grid_zipfile: str,

                        run_id: str):

    """

    Populate the lines table in the DB

    :param config_lines: the dictionary containing the line configuration

    :param engine: the SQLAlchemy engine to use

    :param executed_on: the timestamp of execution, to add to the record

    :param grid_tarfile: the grid tarfile name, to be used as key

    :param grid_zipfile: the grid tarfile name, to be used as key

    :param run_id: the run unique identifier

    """

    line_pars_dict = {

        'zipped_grid_name': f'{grid_tarfile}',

        'zipped_grid_name': f'{grid_zipfile}',

        'lines_mode': config_lines['lines_mode'],

        'created_on': executed_on,

        'run_id': run_id
@@ -188,7 +189,33 @@ def populate_line_table(config_lines: dict, 
    )





def main(grid_tarfile: str,

def populate_stars_table(config_stars: dict,

                         engine: sqlalchemy.engine,

                         executed_on: datetime.timestamp,

                         grid_zipfile: str,

                         run_id: str):

    """

    Populate the stars table in the DB

    :param config_stars: the dictionary containing the stars configurations

    :param engine: the SQLAlchemy engine to use

    :param executed_on: the timestamp of execution, to add to the record

    :param grid_zipfile: the grid tarfile name, to be used as key

    :param run_id: the run unique identifier

    """

    _config_stars = config_stars.copy()

    _config_stars['zipped_grid_name'] = grid_zipfile

    _config_stars['created_on'] = executed_on

    _config_stars['run_id'] = run_id



    upsert(

        table_object=StarsPars,

        row_dict=_config_stars,

        conflict_keys=[StarsPars.zipped_grid_name, StarsPars.run_id],

        engine=engine

    )





def main(grid_zipfile: str,

         run_id: str,

         override_config: Union[dict, None] = None,

         radmc_input_path: Union[str, None] = None,
@@ -224,6 +251,12 @@ def main(grid_tarfile: str, 
        if compute_dust_temperature is True:

            logger.info('Computing dust temperature distribution using the stars in the configuration file')

            os.system('radmc3d mctherm setthreads 16')

            os.system(f'zip -ur {os.path.join(execution_dir, "stg", "archive", grid_zipfile)} dust_temperature.dat')

            populate_stars_table(config_stars=config_stg['stars'],

                                 engine=engine,

                                 executed_on=executed_on,

                                 grid_zipfile=grid_zipfile,

                                 run_id=run_id)

        else:

            logger.info('Using cached dust temperature distribution')

            shutil.copy(os.path.join(execution_dir, 'model', 'data', 'dust_temperature.dat'),
@@ -244,17 +277,17 @@ def main(grid_tarfile: str, 
    populate_line_table(config_lines=config_lines,

                        engine=engine,

                        executed_on=executed_on,

                        grid_tarfile=grid_tarfile,

                        grid_zipfile=grid_zipfile,

                        run_id=run_id)



    populate_species_and_partner_table(config_lines=config_lines,

                                       engine=engine,

                                       executed_on=executed_on,

                                       grid_tarfile=grid_tarfile,

                                       grid_zipfile=grid_zipfile,

                                       run_id=run_id)



    populate_model_table(config_mdl=config_mdl,

                         grid_tarfile=grid_tarfile,

                         grid_zipfile=grid_zipfile,

                         cube_filename=cube_filename,

                         engine=engine,

                         executed_on=executed_on,
@@ -264,5 +297,5 @@ def main(grid_tarfile: str, 




if __name__ == '__main__':

    main(grid_tarfile='459295aa894dffa8c521e606d14dbb6927638a2c.tgz',

    main(grid_zipfile='459295aa894dffa8c521e606d14dbb6927638a2c.zip',

         run_id='test_run')

etl/prs/prs_inspect_results.py

+3 −2
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ from astropy.io import fits 
from sqlalchemy.orm import Session

from sqlalchemy import and_, or_

from stg.stg_build_db_structure import (GridPars,

                                        StarsPars,

                                        ModelPars,

                                        RatioMaps,

                                        MomentZeroMaps)
@@ -33,10 +34,10 @@ def get_aggregated_ratio_from_db( 
    :param session: the SQLAlchemy session to use

    :return: the aggregated ratio

    """

    results = session.query(GridPars, RatioMaps).join(ModelPars).join(MomentZeroMaps).join(RatioMaps,

    results = session.query(GridPars, RatioMaps).join(StarsPars, isouter=True).join(ModelPars).join(MomentZeroMaps).join(RatioMaps,

                                                                                           or_(RatioMaps.mom_zero_map_1,

                                                                                               RatioMaps.mom_zero_map_2)).filter(

        and_(GridPars.dust_temperature == dust_temperature,

        and_(GridPars.dust_temperature_at_reference == dust_temperature,

             GridPars.density_at_reference == gas_density,

             # GridPars.central_density == gas_density,

             or_(ModelPars.iline.in_(lines)))).order_by(GridPars.created_on.desc()).first()

etl/stg/stg_build_db_structure.py

+23 −0
Original line number Diff line number Diff line
@@ -6,6 +6,7 @@ from sqlalchemy import (Column, 
                        String,

                        Float,

                        DateTime,

                        ARRAY,

                        ForeignKeyConstraint)

from assets.commons import (get_pg_engine,

                            setup_logger)
@@ -39,6 +40,7 @@ class GridPars(Base): 
    species_and_partners = relationship("SpeciesAndPartners", cascade="all, delete-orphan")

    fits_cube_name = relationship("ModelPars", cascade="all, delete-orphan")

    line_pars = relationship("LinePars", cascade="all, delete-orphan")

    stars_pars = relationship("StarsPars", cascade="all, delete-orphan")

    grid_type = Column(String)

    coordinate_system = Column(String)

    central_density = Column(Float)
@@ -67,6 +69,27 @@ class GridPars(Base): 
    run_id = Column(String, primary_key=True)





class StarsPars(Base):

    __tablename__ = "stars_parameters"

    __table_args__ = (

        ForeignKeyConstraint(

            ('zipped_grid_name', 'run_id'),

            ['grid_parameters.zipped_grid_name', 'grid_parameters.run_id']

        ),

    )

    zipped_grid_name = Column(String(150), primary_key=True)

    nstars = Column(Integer)

    rstars = Column(ARRAY(Float))

    mstars = Column(ARRAY(Float))

    star_positions = Column(ARRAY(Float))

    star_fluxes = Column(ARRAY(Float))

    nlambdas = Column(Integer)

    spacing = Column(String)

    lambdas_micron_limits = Column(ARRAY(Float))

    created_on = Column(DateTime)

    run_id = Column(String, primary_key=True)





class LinePars(Base):

    __tablename__ = "lines_parameters"

    __table_args__ = (

etl/stg/stg_radmc_input_generator.py

+12 −11
Original line number Diff line number Diff line
@@ -12,7 +12,7 @@ from assets.commons import (compute_power_law_radial_profile, 
                            load_config_file,

                            get_moldata,

                            compute_unique_hash_filename,

                            make_tarfile,

                            make_archive,

                            upsert,

                            get_pg_engine,

                            validate_parameter,
@@ -296,7 +296,7 @@ def remap_metadata_to_grid_row(zipped_grid_name: str, 
                               config: dict) -> dict:

    """

    Create a dictionary that can be persisted in the database, according to the GridPars table structure

    :param zipped_grid_name: the name of the tarball that contains grid files

    :param zipped_grid_name: the name of the compressed archive that contains grid files

    :param grid_metadata: the grid metadata

    :param config: the configuration dictionary, to store human-readable grid- and coordinate types

    :return: the dictionary to be inserted in the database
@@ -364,9 +364,9 @@ def populate_grid_table(config: dict, 
    :param config: the configuration dictionary

    :param engine: the SQLAlchemy engine to use

    :param grid_metadata: the grid metadata

    :return: the tarball filename

    :return: the compressed archive filename

    """

    output_filename = f'{compute_unique_hash_filename(config=config)}.tgz'

    output_filename = f'{compute_unique_hash_filename(config=config)}.zip'

    remapped_row = remap_metadata_to_grid_row(

        zipped_grid_name=output_filename,

        grid_metadata=grid_metadata,
@@ -385,10 +385,10 @@ def populate_grid_table(config: dict, 


def populate_grid_files(quantity: str,

                        engine: sqla_engine,

                        tgz_filename: str,

                        zip_filename: str,

                        filename: str,

                        run_id: str):

    raw_insert_entry = {'zipped_grid_name': tgz_filename,

    raw_insert_entry = {'zipped_grid_name': zip_filename,

                        'quantity': quantity,

                        'fits_grid_name': filename,

                        'created_on': datetime.now(),
@@ -464,7 +464,8 @@ def main(run_id: str, 
                           wavelengths_micron=wavelengths_micron)

    get_moldata(species_names=config['lines']['species_to_include'],

                logger=logger,

                path=input_files_dir)

                path=input_files_dir,

                use_cache=True)

    write_radmc_lines_input(line_config=config['lines'],

                            path=input_files_dir,

                            logger=logger)
@@ -473,7 +474,7 @@ def main(run_id: str, 
                                            grid_metadata=grid_metadata,

                                            line_config=config['lines'],

                                            path=input_files_dir)

    tgz_filename = populate_grid_table(config=config,

    zip_filename = populate_grid_table(config=config,

                                       engine=engine,

                                       grid_metadata=grid_metadata,

                                       run_id=run_id)
@@ -485,14 +486,14 @@ def main(run_id: str, 


    populate_grid_files(quantity='gas_number_density',

                        engine=engine,

                        tgz_filename=tgz_filename,

                        zip_filename=zip_filename,

                        filename=grid_file_name,

                        run_id=run_id)



    make_tarfile(output_filename=tgz_filename,

    make_archive(output_filename=zip_filename,

                 source_dir=input_files_dir,

                 archive_path=os.path.join('stg', 'archive'))

    return tgz_filename

    return zip_filename





if __name__ == '__main__':