Commit a686cf30 authored by jay's avatar jay
Browse files

comments

parent d456c369

autocnet/__init__.py

+3 −2
Original line number Diff line number Diff line
@@ -23,13 +23,15 @@ except DistributionNotFound: 
else:

    __version__ = _dist.version



# Defaults

dem = None



from config_parser import parse_config



config = parse_config()



if config:

    dem = config['spatial']['dem']

    if dem:

    try:

        dem = GeoDataset(dem)

    except:
@@ -47,7 +49,6 @@ if config: 
                    isolation_level="AUTOCOMMIT")                   

    Session = orm.session.sessionmaker(bind=engine)

else:

    dem = None

    def sessionwarn():

        raise RuntimeError('This call requires a database connection.')

    Session = sessionwarn

autocnet/spatial/overlap.py

+20 −8
Original line number Diff line number Diff line
@@ -7,7 +7,7 @@ import shapely 
import sqlalchemy

from plio.io.io_gdal import GeoDataset



from autocnet import config, dem

from autocnet import config, dem, Session

from autocnet.cg import cg as compgeom

from autocnet.io.db.model import Images, Measures, Overlay, Points, JsonEncoder

from autocnet.spatial import isis
@@ -85,9 +85,6 @@ def cluster_place_points_in_overlaps(size_threshold=0.0007, 
               options: {"csm", "isis"}

               Pick what kind of camera model implementation to use

    """

    # Get all of the overlaps over the size threshold

    overlaps = Overlay.overlapping_larger_than(size_threshold)



    # Setup the redis queue

    rqueue = StrictRedis(host=config['redis']['host'],

                         port=config['redis']['port'],
@@ -95,13 +92,28 @@ def cluster_place_points_in_overlaps(size_threshold=0.0007, 


    # Push the job messages onto the queue

    queuename = config['redis']['processing_queue']

    for overlap in overlaps:

        msg = {'id' : overlap.id,

    past = 0

    session = Session()

    ids = session.query(Overlay.id).all()

    session.close()

    for i, id in enumerate(ids):

        msg = {'id' : id[0],

               'distribute_points_kwargs' : distribute_points_kwargs,

               'walltime' : walltime,

               'cam_type': cam_type}

        rqueue.rpush(queuename, json.dumps(msg, cls=JsonEncoder))

    job_counter = len([*overlaps]) + 1

        if (i > 0) and i % 10000 == 0:

            job_counter = i - past + 1

            past = i

            # Submit the jobs

            submitter = Slurm('acn_overlaps',

                        mem_per_cpu=config['cluster']['processing_memory'],

                        time=walltime,

                        partition=config['cluster']['queue'],

                        output=config['cluster']['cluster_log_dir']+'/autocnet.place_points-%j')

            submitter.submit(array='1-{}'.format(job_counter))



    job_counter = i - past + 1



    # Submit the jobs

    submitter = Slurm('acn_overlaps',
@@ -110,7 +122,7 @@ def cluster_place_points_in_overlaps(size_threshold=0.0007, 
                 partition=config['cluster']['queue'],

                 output=config['cluster']['cluster_log_dir']+'/autocnet.place_points-%j')

    submitter.submit(array='1-{}'.format(job_counter))

    return job_counter

    return i + 1



def place_points_in_overlap(nodes, geom, cam_type="csm",

                            distribute_points_kwargs={}):

config_parser.py

+3 −0
Original line number Diff line number Diff line
@@ -17,6 +17,9 @@ def parse_config(name='autocnet_config'): 
    spatial = config.get('spatial', None)

    if spatial == None:

        raise KeyError('Config is missing the root "spatial" key.')

    for k in ['latitudinal_srid']:

        if k not in spatial.keys():

            raise KeyError(f'Missing key: {k} in the spatial section of the config.')

            

    database = config.get('database', None)

    if database == None: