Cluster load images

                                     distribute_points_kwargs={},

                                     walltime='00:10:00',

                                     chunksize=1000,

                                     cam_type="csm"):

                                     cam_type="csm",

                                     query_string='SELECT overlay.id FROM overlay LEFT JOIN points ON ST_INTERSECTS(overlay.geom, points.geom) WHERE points.id IS NULL;'):

    """

    Place points in all of the overlap geometries by back-projecing using

    sensor models. This method uses the cluster to process all of the overlaps

    cam_type : str

               options: {"csm", "isis"}

               Pick what kind of camera model implementation to use

    query

    """

    # Setup the redis queue

    rqueue = StrictRedis(host=config['redis']['host'],

    queuename = config['redis']['processing_queue']

    past = 0

    session = Session()

    ids = session.query(Overlay.id).all()

    ids = [i[0] for i in session.execute(query_string)]

    session.close()

    for i, id in enumerate(ids):

        msg = {'id' : id[0],

        msg = {'id' : id,

               'distribute_points_kwargs' : distribute_points_kwargs,

               'walltime' : walltime,

               'cam_type': cam_type}

#!/usr/bin/env python

import json

import os

os.environ['PROJ_LIB'] = '/home/jlaura/anaconda3/envs/autocnet/share/proj'

import sys

import time

import warnings

import csmapi

from knoten.csm import generate_latlon_footprint, generate_boundary

from plio.io.io_gdal import GeoDataset

from plio.io.isis_serial_number import generate_serial_number

import pvl

from redis import StrictRedis

import requests

from requests.adapters import HTTPAdapter

from requests.packages.urllib3.util.retry import Retry

import shapely

import yaml

from autocnet.io.db.redis_queue import pop_computetime_push

from autocnet.io.db.model import Images, Cameras

from autocnet.utils import utils

def requests_retry_session(

    retries=3,

    backoff_factor=0.3,

    status_forcelist=(500, 502, 504),

    session=None,

):

    session = session or requests.Session()

    retry = Retry(

        total=retries,

        read=retries,

        connect=retries,

        backoff_factor=backoff_factor,

        status_forcelist=status_forcelist,

    )

    adapter = HTTPAdapter(max_retries=retry)

    session.mount('http://', adapter)

    session.mount('https://', adapter)

    return session

#Load the config file

try:

    print('Using config: ', os.environ['autocnet_config'])

    with open(os.environ['autocnet_config'], 'r') as f:

        config = yaml.safe_load(f)

except:

    print("The 'autocnet_config' environment variable is not set.")

    sys.exit(1)

def create_footprint(config, geodata, camera):

    boundary = generate_boundary(geodata.raster_size[::-1])  # yx to xy

    dem = GeoDataset(config['spatial']['dem'])

    footprint_latlon = generate_latlon_footprint(camera, boundary, dem=dem)

    footprint_latlon.FlattenTo2D()

    return footprint_latlon

def create_camera(config, geodata, imagepath):

    # Create the camera entry

    label = pvl.dumps(geodata.metadata).decode()

    url = config['pfeffernusse']['url']

    response = requests_retry_session.post(url, json={'label':label})

    response = response.json()

    model_name = response.get('name_model', None)

    if model_name is None:

        return (None, None)

    isdpath = os.path.splitext(imagepath)[0] + '.json'

    try:

        with open(isdpath, 'w') as f:

            json.dump(response, f)

    except Exception as e:

        warnings.warn('Failed to write JSON ISD for image {}.\n{}'.format(imagepath, e))

    isd = csmapi.Isd(imagepath)

    plugin = csmapi.Plugin.findPlugin('UsgsAstroPluginCSM')

    camera = plugin.constructModelFromISD(isd, model_name)

    serialized_camera = camera.getModelState()

    cam = Cameras(camera=serialized_camera)

    return cam, camera

def main(msg, config):

    images = [] 

    for path in msg['imagepaths']:

        print(f'Processing: {path}')

        try:

            geodata = GeoDataset(path)

            dbcam, cam = create_camera(config, geodata, path)

            if dbcam is None:

                warnings.warn(f'Failed to add {path}')

                continue

            fp = create_footprint(config, geodata, cam)

            if isinstance(fp, shapely.geometry.Polygon):

                fp = shapely.geometry.MultiPolygon([fp])

            serial = generate_serial_number(path)

            i = Images(name=geodata.file_name,

                    path=path,

                    footprint_latlon=fp,

                    cameras=dbcam,

                    serial=serial)

            images.append(i)

        except:

            warnings.warn(f'Failed to add {path}.')

    Images.bulkadd(images)

if __name__ == '__main__':

    conf = config['redis']

    queue = StrictRedis(host=conf['host'], port=conf['port'], db=0)

    msg = pop_computetime_push(queue,

                               conf['processing_queue'],

                               conf['working_queue'])

    main(msg, config)

def main(msg, config):

    session = Session()

    id = msg['id']

    res = session.query(Overlay).filter(Overlay.id == msg['id'])

    if res is None:

    oid = msg['id']

    overlap = session.query(Overlay).filter(Overlay.id == oid).one()

    if overlap is None:

        print('Could not find overlap with ID', id)

        sys.exit(1)

    overlap = res.first()

    geom = overlap.geom

    nodes = []

    for id in overlap.intersections:

        res = session.query(Images).filter(Images.id == id).first()

        res = session.query(Images).filter(Images.id == id).one()

        nodes.append(NetworkNode(node_id=id, image_path=res.path))

    session.close()

    print('Placing points in overlap', id)

    print('Placing points in overlap', oid)

    points = place_points_in_overlap(nodes, geom, msg["cam_type"],

                                     msg['distribute_points_kwargs'])