Merge pull request #373 from jlaura/serialize

import json

import math

import os

from shutil import copyfile

from time import gmtime, strftime, time

import warnings

        return obj

    def copy_images(self, newdir):

        """

        Copy images from a given directory into a new directory and

        update the 'path' column in the Images table.

        Parameters

        ----------

        newdir : str

                 The full output PATH where the images are to be copied to.

        """

        if not os.path.exists(newdir):

            os.makedirs(newdir)

        session = Session()

        images = session.query(Images).all()

        oldnew = []

        for obj in images:

            oldpath = obj.path

            filename = os.path.basename(oldpath)

            obj.path = os.path.join(newdir, filename)

            oldnew.append((oldpath, obj.path))

        session.commit()

        session.close()

        # Copy the files

        [copyfile(old, new) for old, new in oldnew]

    @classmethod

    def from_remote_database(cls, source_db_config, path,  query_string='SELECT * FROM public.images LIMIT 10'):

        """

        This is a constructor that takes an existing database containing images and sensors, 

        copies the selected rows into the project specified in the autocnet_config variable, 

        and instantiates a new NetworkCandidateGraph object. This method is

        similar to the `from_database` method. The main difference is that this

        method assumes that the image and sensor rows are prepopulated in an external db

        and simply copies those entires into the currently speficied project.

        Currently, this method does NOT check for duplicate serial numbers during the 

        bulk add. Therefore multiple runs of this method on the same database will fail.

        Parameters

        ----------

        source_db_config : dict

                           In the form: {'username':'somename',

                                         'password':'somepassword',

                                         'host':'somehost',

                                         'pgbouncer_port':6543,

                                         'name':'somename'}

        path : str

               The PATH to which images in the database specified in the config

               will be copied to. This method duplicates the data and copies it

               to a user defined PATH to avoid issues with updating image ephemeris

               across projects.

        query_string : str

                       An optional string to select a subset of the images in the 

                       database specified in the config. 

        Returns

        -------

        obj : obj

              A network candidate graph.

        Example

        -------

        >>> source_db_config = {'username':'jay',

        'password':'abcde',

        'host':'autocnet.wr.usgs.gov',

        'pgbouncer_port':5432,

        'name':'ctx'}

        >>> geom = 'LINESTRING(145 10, 145 11, 146 11, 146 10, 145 10)'

        >>> srid = 949900

        >>> outpath = '/scratch/jlaura/fromdb'

        >>> query = f"SELECT * FROM Images WHERE ST_INTERSECTS(footprint_latlon, ST_Polygon(ST_GeomFromText('{geom}'), {srid})) = TRUE"

        >>> ncg = NetworkCandidateGraph.from_remote_database(source_db_config, outpath, query_string=query)

        """

        sourceSession, _ = new_connection(source_db_config)

        sourcesession = sourceSession()

        sourceimages = sourcesession.execute(query_string).fetchall()

        destinationsession = Session()

        destinationsession.execute(Images.__table__.insert(), sourceimages)

        # Get the camera objects to manually join. Keeps the caller from

        # having to remember to bring cameras as well.

        ids = [i[0] for i in sourceimages]

        cameras = sourcesession.query(Cameras).filter(Cameras.image_id.in_(ids)).all()

        for c in cameras:

            destinationsession.merge(c)

        destinationsession.commit()

        destinationsession.close()

        sourcesession.close()

        # Create the graph, copy the images, and compute the overlaps

        obj = cls.from_database()

        obj.copy_images(path)

        obj._execute_sql(compute_overlaps_sql)

        return obj

    @classmethod

    def from_database(cls, query_string='SELECT * FROM public.images'):

        """

        super(NetworkNode, self).__init__(*args, **kwargs)

        # If this is the first time that the image is seen, add it to the DB

        if parent is None:

            self.parent = Parent(config)

            self.parent = Parent(config['database'])

        else:

            self.parent = parent

class Parent:

    def __init__(self, config):

        self.session, _ = new_connection(config)

        Session, _ = new_connection(config)

        self.session = Session()

        self.session.begin()

def new_connection(config):

def new_connection(dbconfig):

    """

    Using the user supplied config create a NullPool database connection.

    engine : object

             An SQLAlchemy engine object

    """

    db = config['database']

    db_uri = 'postgresql://{}:{}@{}:{}/{}'.format(db['username'],

                                                  db['password'],

                                                  db['host'],

                                                  db['pgbouncer_port'],

                                                  db['name'])    

    db_uri = 'postgresql://{}:{}@{}:{}/{}'.format(dbconfig['username'],

                                                  dbconfig['password'],

                                                  dbconfig['host'],

                                                  dbconfig['pgbouncer_port'],

                                                  dbconfig['name'])    

    engine = sqlalchemy.create_engine(db_uri,

                                      poolclass=sqlalchemy.pool.NullPool)

    Session = sqlalchemy.orm.sessionmaker(bind=engine, autocommit=True)

    return Session(), engine

    return Session, engine

                                     distribute_points_kwargs={},

                                     walltime='00:10:00',

                                     chunksize=1000,

                                     cam_type="csm"):

                                     cam_type="csm",

                                     query_string='SELECT overlay.id FROM overlay LEFT JOIN points ON ST_INTERSECTS(overlay.geom, points.geom) WHERE points.id IS NULL;'):

    """

    Place points in all of the overlap geometries by back-projecing using

    sensor models. This method uses the cluster to process all of the overlaps

    cam_type : str

               options: {"csm", "isis"}

               Pick what kind of camera model implementation to use

    query

    """

    # Setup the redis queue

    rqueue = StrictRedis(host=config['redis']['host'],

    queuename = config['redis']['processing_queue']

    past = 0

    session = Session()

    ids = session.query(Overlay.id).all()

    ids = [i[0] for i in session.execute(query_string)]

    session.close()

    for i, id in enumerate(ids):

        msg = {'id' : id[0],

        msg = {'id' : id,

               'distribute_points_kwargs' : distribute_points_kwargs,

               'walltime' : walltime,

               'cam_type': cam_type}

#!/usr/bin/env python

import json

import os

os.environ['PROJ_LIB'] = '/home/jlaura/anaconda3/envs/autocnet/share/proj'

import sys

import time

import warnings

import csmapi

from knoten.csm import generate_latlon_footprint, generate_boundary

from plio.io.io_gdal import GeoDataset

from plio.io.isis_serial_number import generate_serial_number

import pvl

from redis import StrictRedis

import requests

from requests.adapters import HTTPAdapter

from requests.packages.urllib3.util.retry import Retry

import shapely

import yaml

from autocnet.io.db.redis_queue import pop_computetime_push

from autocnet.io.db.model import Images, Cameras

from autocnet.utils import utils

from autocnet import Session

def requests_retry_session(

    retries=3,

    backoff_factor=0.3,

    status_forcelist=(500, 502, 504),

    session=None,

):

    session = session or requests.Session()

    retry = Retry(

        total=retries,

        read=retries,

        connect=retries,

        backoff_factor=backoff_factor,

        status_forcelist=status_forcelist,

    )

    adapter = HTTPAdapter(max_retries=retry)

    session.mount('http://', adapter)

    session.mount('https://', adapter)

    return session

#Load the config file

try:

    print('Using config: ', os.environ['autocnet_config'])

    with open(os.environ['autocnet_config'], 'r') as f:

        config = yaml.safe_load(f)

except:

    print("The 'autocnet_config' environment variable is not set.")

    sys.exit(1)

def create_footprint(config, geodata, camera):

    boundary = generate_boundary(geodata.raster_size[::-1])  # yx to xy

    dem = GeoDataset(config['spatial']['dem'])

    footprint_latlon = generate_latlon_footprint(camera, boundary, dem=dem)

    footprint_latlon.FlattenTo2D()

    return footprint_latlon

def create_camera(config, geodata, imagepath):

    # Create the camera entry

    label = pvl.dumps(geodata.metadata).decode()

    url = config['pfeffernusse']['url']

    response = requests_retry_session().post(url, json={'label':label})

    response = response.json()

    model_name = response.get('name_model', None)

    if model_name is None:

        return (None, None)

    isdpath = os.path.splitext(imagepath)[0] + '.json'

    with open(isdpath, 'w') as f:

        json.dump(response, f)

    isd = csmapi.Isd(imagepath)

    plugin = csmapi.Plugin.findPlugin('UsgsAstroPluginCSM')

    camera = plugin.constructModelFromISD(isd, model_name)

    serialized_camera = camera.getModelState()

    cam = Cameras(camera=serialized_camera)

    return cam, camera

def main(msg, config):

    session = Session()

    serials = [s[0] for s in session.query(Images.serial).all()]

    session.close()

    images = [] 

    for path in msg['imagepaths']:

        try:

            serial = generate_serial_number(path)

        except:

            warnings.warn(f'Unable to generate serial for {path}')

            continue

        if serial in serials:

            print(f'Image {path} already in database.')

            continue

        print(f'Processing: {path}')

        try:

            geodata = GeoDataset(path)

            dbcam, cam = create_camera(config, geodata, path)

            if dbcam is None:

                warnings.warn(f'Failed to add {path}')

                continue

            fp = create_footprint(config, geodata, cam)

            if isinstance(fp, shapely.geometry.Polygon):

                fp = shapely.geometry.MultiPolygon([fp])

            serial = generate_serial_number(path)

            i = Images(name=geodata.file_name,

                    path=path,

                    footprint_latlon=fp,

                    cameras=dbcam,

                    serial=serial)

            images.append(i)

        except:

            warnings.warn(f'Failed to add {path}.')

    Images.bulkadd(images)

if __name__ == '__main__':

    conf = config['redis']

    queue = StrictRedis(host=conf['host'], port=conf['port'], db=0)

    msg = pop_computetime_push(queue,

                               conf['processing_queue'],

                               conf['working_queue'])

    main(msg, config)