Commit 6df56fd2 authored by Lauren Adoram-Kershner's avatar Lauren Adoram-Kershner Committed by jlaura
Browse files

Adding a node exclusion option to cluster jobs (#388) 

* Adding a node exclusion option to cluster jobs

* updating documentation
parent 89354eb1

autocnet/matcher/subpixel.py

+7 −2
Original line number Diff line number Diff line
@@ -495,7 +495,8 @@ def cluster_subpixel_register_points(iterative_phase_kwargs={'size': 251}, 
                                     cost_func=lambda x,y: 1/x**2 * y,

                                     threshold=0.005,

                                     walltime='00:10:00',

                                     chunksize=1000):

                                     chunksize=1000,

                                     exclude=None):

    """

    Distributed subpixel registration of all of the points in a given DB table.
@@ -520,6 +521,10 @@ def cluster_subpixel_register_points(iterative_phase_kwargs={'size': 251}, 
    threshold : numeric

                measures with a cost <= the threshold are marked as active=False in

                the database.



    exclude : str

              string containing the name(s) of any slurm nodes to exclude when

              completing a cluster job. (e.g.: 'gpu1' or 'gpu1,neb12')

    """

    # Setup the redis queue

    rqueue = StrictRedis(host=config['redis']['host'],
@@ -549,5 +554,5 @@ def cluster_subpixel_register_points(iterative_phase_kwargs={'size': 251}, 
                 time=walltime,

                 partition=config['cluster']['queue'],

                 output=config['cluster']['cluster_log_dir']+f'/autocnet.subpixel_register-%j')

    submitter.submit(array='1-{}'.format(job_counter), chunksize=chunksize)

    submitter.submit(array='1-{}'.format(job_counter), chunksize=chunksize, exclude=exclude)

    return job_counter

autocnet/spatial/overlap.py

+7 −2
Original line number Diff line number Diff line
@@ -69,6 +69,7 @@ def cluster_place_points_in_overlaps(size_threshold=0.0007, 
                                     distribute_points_kwargs={},

                                     walltime='00:10:00',

                                     chunksize=1000,

                                     exclude=None,

                                     cam_type="csm",

                                     query_string='SELECT overlay.id FROM overlay LEFT JOIN points ON ST_INTERSECTS(overlay.geom, points.geom) WHERE points.id IS NULL;'):

    """
@@ -88,7 +89,11 @@ def cluster_place_points_in_overlaps(size_threshold=0.0007, 
               options: {"csm", "isis"}

               Pick what kind of camera model implementation to use



    query

    query : str



    exclude : str

              string containing the name(s) of any slurm nodes to exclude when

              completing a cluster job. (e.g.: 'gpu1' or 'gpu1,neb12')

    """

    # Setup the redis queue

    rqueue = StrictRedis(host=config['redis']['host'],
@@ -115,7 +120,7 @@ def cluster_place_points_in_overlaps(size_threshold=0.0007, 
                 partition=config['cluster']['queue'],

                 output=config['cluster']['cluster_log_dir']+'/autocnet.place_points-%j')

    job_counter = i+1

    submitter.submit(array='1-{}'.format(job_counter), chunksize=chunksize)

    submitter.submit(array='1-{}'.format(job_counter), chunksize=chunksize, exclude=exclude)

    return job_counter



def place_points_in_overlap(nodes, geom, cam_type="csm",