Commit 30c34463 authored by jlaura's avatar jlaura Committed by GitHub
Browse files

Abstracts out geoalchemy to single place (#303) 

* Abstracts out geoalchemy to single place

* add ipykernel to environment set up

* add jupyter to environment

* Fixes for properties

* updates for failing tests

* propery on measure type

* Abstracts out geoalchemy to single place

* Fixes for properties

* updates for failing tests

* propery on measure type

* Adds shapely serialization to WKT and JSONEncoder tests
parent 0aecb0d3

autocnet/graph/edge.py

+4 −3
Original line number Diff line number Diff line
@@ -2,7 +2,6 @@ from functools import wraps, singledispatch 
import warnings

from collections import defaultdict, MutableMapping, Counter



from geoalchemy2.elements import WKBElement

import numpy as np

import pandas as pd

import networkx as nx
@@ -888,8 +887,10 @@ class NetworkEdge(Edge): 
                        row_val = int(row_val)

                    elif isinstance(row_val, (np.float,)):

                        row_val = float(row_val)

                    elif isinstance(row_val, WKBElement):

                        continue

                    # This should be uncommented if the matches

                    # df is refactored to be a geodataframe

                    #elif isinstance(row_val, WKBElement):

                    #    continue

                    mapping[index] = row_val

                to_db_update.append(mapping)

            else:

autocnet/graph/network.py

+0 −2
Original line number Diff line number Diff line
@@ -10,8 +10,6 @@ import networkx as nx 
import geopandas as gpd

import pandas as pd

import numpy as np

import sqlalchemy

import geoalchemy2

from redis import StrictRedis



import shapely.affinity

autocnet/graph/node.py

+1 −5
Original line number Diff line number Diff line
@@ -4,7 +4,6 @@ import os 
import warnings



from csmapi import csmapi

import geoalchemy2

import numpy as np

import pandas as pd

from plio.io.io_gdal import GeoDataset
@@ -653,11 +652,8 @@ class NetworkNode(Node): 
            boundary = generate_boundary(self.geodata.raster_size[::-1])  # yx to xy

            footprint_latlon = generate_latlon_footprint(self.camera, boundary)

            footprint_latlon.FlattenTo2D()

            footprint_latlon = footprint_latlon.ExportToWkt()

            footprint_latlon = geoalchemy2.elements.WKTElement(footprint_latlon,

                                                               srid=config['spatial']['srid'])

        else:

            footprint_latlon = geoalchemy2.shape.to_shape(res.footprint_latlon)

            footprint_latlon = res.footprint_latlon

        return footprint_latlon



    @property

autocnet/io/db/model.py

+81 −8
Original line number Diff line number Diff line
@@ -12,10 +12,13 @@ from sqlalchemy.dialects.postgresql import ARRAY, JSONB 
from sqlalchemy.orm import relationship, backref

from sqlalchemy_utils import database_exists, create_database

from sqlalchemy.types import TypeDecorator

from sqlalchemy.ext.hybrid import hybrid_property



from geoalchemy2 import Geometry

from geoalchemy2.shape import to_shape

from geoalchemy2.shape import from_shape, to_shape



import osgeo

import shapely

from autocnet import engine, Session, config



Base = declarative_base()
@@ -44,6 +47,8 @@ class JsonEncoder(json.JSONEncoder): 
            return obj.decode("utf-8")

        if isinstance(obj, set):

            return list(obj)

        if isinstance(obj,  shapely.geometry.base.BaseGeometry):

            return obj.wkt

        return json.JSONEncoder.default(self, obj)



class IntEnum(TypeDecorator):
@@ -56,7 +61,9 @@ class IntEnum(TypeDecorator): 
        self._enumtype = enumtype



    def process_bind_param(self, value, dialect):

        return value.value

        if hasattr(value, 'value'):

            value = value.value

        return value



    def process_result_value(self, value, dialect):

        return self._enumtype(value)
@@ -151,7 +158,7 @@ class Matches(BaseMixin, Base): 
    destination_idx = Column(Integer, nullable=False)

    lat = Column(Float)

    lon = Column(Float)

    geom = Column(Geometry('POINT', dimension=2, srid=srid, spatial_index=True))

    _geom = Column("geom", Geometry('POINT', dimension=2, srid=srid, spatial_index=True))

    source_x = Column(Float)

    source_y = Column(Float)

    destination_x = Column(Float)
@@ -161,6 +168,17 @@ class Matches(BaseMixin, Base): 
    original_destination_x = Column(Float)

    original_destination_y = Column(Float)



    @hybrid_property

    def geom(self):

        try:

            return to_shape(self._geom)

        except:

            return self._geom



    @geom.setter

    def geom(self, geom):

        if geom:  # Supports instances where geom is explicitly set to None.

            self._geom = from_shape(geom, srid=srid)



class Cameras(BaseMixin, Base):

    __tablename__ = 'cameras'
@@ -176,7 +194,7 @@ class Images(BaseMixin, Base): 
    path = Column(String)

    serial = Column(String, unique=True)

    active = Column(Boolean)

    footprint_latlon = Column(Geometry('MultiPolygon', srid=srid, dimension=2, spatial_index=True))

    _footprint_latlon = Column("footprint_latlon", Geometry('MultiPolygon', srid=srid, dimension=2, spatial_index=True))

    footprint_bodyfixed = Column(Geometry('MULTIPOLYGON', dimension=2))

    #footprint_bodyfixed = Column(Geometry('POLYGON',dimension=3))
@@ -196,13 +214,36 @@ class Images(BaseMixin, Base): 
                'footprint_latlon':footprint,

                'footprint_bodyfixed':self.footprint_bodyfixed})



    @hybrid_property

    def footprint_latlon(self):

        try:

            return to_shape(self._footprint_latlon)

        except:

            return self._footprint_latlon



    @footprint_latlon.setter

    def footprint_latlon(self, geom):

        if isinstance(geom, osgeo.ogr.Geometry):

            # If an OGR geom, convert to shapely

            geom = shapely.wkt.loads(geom.ExportToWkt())

        self._footprint_latlon = from_shape(geom, srid=srid)



class Overlay(BaseMixin, Base):

    __tablename__ = 'overlay'

    id = Column(Integer, primary_key=True, autoincrement=True)

    intersections = Column(ARRAY(Integer))

    #geom = Column(Geometry(geometry_type='POLYGON', management=True))  # sqlite

    geom = Column(Geometry('POLYGON', srid=srid, dimension=2, spatial_index=True))  # postgresql

    _geom = Column("geom", Geometry('POLYGON', srid=srid, dimension=2, spatial_index=True))  # postgresql



    @hybrid_property

    def geom(self):

        try:

            return to_shape(self._geom)

        except:

            return self._geom

    @geom.setter

    def geom(self, geom):

        self._geom = from_shape(geom, srid=srid)



class PointType(enum.IntEnum):

    """
@@ -215,9 +256,9 @@ class PointType(enum.IntEnum): 
class Points(BaseMixin, Base):

    __tablename__ = 'points'

    id = Column(Integer, primary_key=True, autoincrement=True)

    pointtype = Column(IntEnum(PointType), nullable=False)  # 2, 3, 4 - Could be an enum in the future, map str to int in a decorator

    _pointtype = Column("pointtype", IntEnum(PointType), nullable=False)  # 2, 3, 4 - Could be an enum in the future, map str to int in a decorator

    identifier = Column(String, unique=True)

    geom = Column(Geometry('POINT', srid=srid, dimension=2, spatial_index=True))

    _geom = Column("geom", Geometry('POINT', srid=srid, dimension=2, spatial_index=True))

    active = Column(Boolean, default=True)

    apriorix = Column(Float)

    aprioriy = Column(Float)
@@ -228,6 +269,28 @@ class Points(BaseMixin, Base): 
    measures = relationship('Measures')

    rms = Column(Float)



    @hybrid_property

    def geom(self):

        try:

            return to_shape(self._geom)

        except:

            return self._geom



    @geom.setter

    def geom(self, geom):

        if geom:

            self._geom = from_shape(geom, srid=srid)



    @hybrid_property

    def pointtype(self):

        return self._pointtype



    @pointtype.setter

    def pointtype(self, v):

        if isinstance(v, int):

            v = PointType(v)

        self._pointtype = v

        

class MeasureType(enum.IntEnum):

    """

    Enum to enforce measure type for ISIS control networks
@@ -243,7 +306,7 @@ class Measures(BaseMixin, Base): 
    pointid = Column(Integer, ForeignKey('points.id'), nullable=False)

    imageid = Column(Integer, ForeignKey('images.id'))

    serial = Column(String, nullable=False)

    measuretype = Column(IntEnum(MeasureType), nullable=False)  # [0,3]  # Enum as above

    _measuretype = Column("measuretype", IntEnum(MeasureType), nullable=False)  # [0,3]  # Enum as above

    sample = Column(Float, nullable=False)

    line = Column(Float, nullable=False)

    sampler = Column(Float)  # Sample Residual
@@ -256,6 +319,16 @@ class Measures(BaseMixin, Base): 
    linesigma = Column(Float)

    rms = Column(Float)



    @hybrid_property

    def measuretype(self):

        return self._measuretype



    @measuretype.setter

    def measuretype(self, v):

        if isinstance(v, int):

            v = MeasureType(v)

        self._measuretype = v



if Session:

    from autocnet.io.db.triggers import valid_point_function, valid_point_trigger

    # Create the database

autocnet/io/db/tests/test_model.py

+54 −0
Original line number Diff line number Diff line 
from datetime import datetime

import json



import numpy as np

import pytest

import sqlalchemy

from shapely.geometry import Polygon, Point



from autocnet.io.db import model

from autocnet import Session, engine
@@ -50,6 +55,13 @@ def test_create_camera(session): 
    res = session.query(model.Cameras).first()

    assert c.id == res.id



def test_create_camera_unique_constraint(session):

    model.Images.create(session, **{'id':1})

    data = {'image_id':1}

    model.Cameras.create(session, **data)

    with pytest.raises(sqlalchemy.exc.IntegrityError):

        model.Cameras.create(session, **data)



def test_images_exists(tables):

    assert model.Images.__tablename__ in tables
@@ -78,8 +90,50 @@ def test_create_images_constrined(session, data): 
def test_overlay_exists(tables):

    assert model.Overlay.__tablename__ in tables



@pytest.mark.parametrize('data', [

    {'id':1},

    {'id':1, 'intersections':[1,2,3]},

    {'id':1, 'intersections':[1,2,3],

     'geom':Polygon([(0,0), (1,0), (1,1), (0,1), (0,0)])}



])

def test_create_overlay(session, data):

    d = model.Overlay.create(session, **data)

    resp = session.query(model.Overlay).filter(model.Overlay.id == d.id).first()

    assert d == resp



def test_points_exists(tables):

    assert model.Points.__tablename__ in tables



@pytest.mark.parametrize("data", [

    {'id':1, 'pointtype':2},

    {'pointtype':2, 'identifier':'123abc'},

    {'pointtype':3, 'geom':Point(0,0)},

    {'pointtype':2, 'geom':Point(1,1), 'active':True},

    {'pointtype':2, 'rms':0.001}

])

def test_create_point(session, data):

    p = model.Points.create(session, **data)

    resp = session.query(model.Points).filter(model.Points.id == p.id).first()

    assert p == resp



def test_measures_exists(tables):

    assert model.Measures.__tablename__ in tables



@pytest.mark.parametrize("data, serialized", [

    ({'foo':np.arange(5)}, {"foo": [0, 1, 2, 3, 4]}),

    ({'foo':np.int64(1)}, {"foo": 1}),

    ({'foo':b'bar'}, {"foo": "bar"}),

    ({'foo':set(['a', 'b', 'c'])}, {"foo": ["a", "b", "c"]}),

    ({'foo':Point(0,0)}, {"foo": 'POINT (0 0)'}),

    ({'foo':datetime(1982, 9, 8)}, {"foo": '1982-09-08 00:00:00'})



])

def test_json_encoder(data, serialized):

    res = json.dumps(data, cls=model.JsonEncoder)

    res = json.loads(res)

    if isinstance(res['foo'], list):

        res['foo'] = sorted(res['foo'])

    print(res)



    assert res == serialized
 
 No newline at end of file