Adds in automatic covariance computation on writes (#454) (602540a1) · Commits · aflab / astrogeology / Autocnet

autocnet/control/control.py

+40 −0

Original line number	Diff line number	Diff line
		@@ -8,7 +8,47 @@ from shapely.geometry import Point
		from autocnet.matcher import subpixel as sp

		from plio.io.io_controlnetwork import to_isis, write_filelist
		from plio.utils import covariance

		def compute_covariance(df, latsigma, lonsigma, radsigma, radius):
		"""
		Compute the covariance matrices for constrained or fixed points.

		Parameters
		----------
		df : pd.DataFrame
		with columns pointtype, adjustedY, and adjustedX

		latsigma : int/float
		The estimated sigma (error) in the latitude direction

		lonsigma : int/float
		The estimated sigma (error) in the longitude direction

		radsigma : int/float
		The estimated sigma (error) in the radius direction

		radius : int/float
		The body semimajor radius
		"""
		def compute_covar(row, latsigma, lonsigma, radsigma, radius):
		if row['pointtype'] == 3 or row['pointtype'] == 4:
		return covariance.compute_covariance(row['adjustedY'],
		row['adjustedX'],
		radius,
		latsigma=latsigma,
		lonsigma=lonsigma,
		radsigma=radsigma,
		semimajor_axis=radius)
		return []

		df['aprioriCovar'] = df.apply(compute_covar,
		axis=1,
		args=(latsigma,
		lonsigma,
		radsigma,
		radius))
		return df

		def identify_potential_overlaps(cg, cn, overlap=True):
		"""

autocnet/control/tests/test_control.py

+13 −0

Original line number	Diff line number	Diff line
		@@ -31,6 +31,19 @@ def test_potential_overlap(controlnetwork, candidategraph):
		(0,), (0,)],
		index=[6,7,8,9,10,11]))

		def test_compute_covariance():
		df = pd.DataFrame([[0,0,3], [0,0,4], [0,0,2]], columns=['adjustedY', 'adjustedX', 'pointtype'])
		df = control.compute_covariance(df, 10, 10, 15, 100)

		def assertexists(row):
		if row['pointtype'] > 2:
		assert row['aprioriCovar'].sum() > 100
		else:
		assert row['aprioriCovar'] == []

		df.apply(assertexists, axis=1)


		"""
		def test_fromcandidategraph(candidategraph, controlnetwork_data):#, controlnetwork):
		matches = candidategraph.get_matches()

autocnet/graph/network.py

+63 −91

Original line number	Diff line number	Diff line
		@@ -13,12 +13,13 @@ import geopandas as gpd
		import pandas as pd
		import numpy as np
		from redis import StrictRedis
		import shapely

		import geoalchemy2
		from sqlalchemy.ext.declarative.api import DeclarativeMeta
		import shapely.affinity
		import shapely.geometry
		import shapely.wkt as swkt
		import shapely.wkb as swkb
		import shapely.ops

		from plio.io.io_controlnetwork import to_isis, from_isis
		@@ -28,6 +29,7 @@ from plio.io.io_gdal import GeoDataset
		from plio.io.isis_serial_number import generate_serial_number
		from plio.io import io_controlnetwork as cnet


		from plurmy import Slurm

		import autocnet
		@@ -37,6 +39,7 @@ from autocnet.graph import markov_cluster
		from autocnet.graph.edge import Edge, NetworkEdge
		from autocnet.graph.node import Node, NetworkNode
		from autocnet.io import network as io_network
		from autocnet.io.db import controlnetwork as io_controlnetwork
		from autocnet.io.db.model import (Images, Keypoints, Matches, Cameras, Points,
		Base, Overlay, Edges, Costs, Measures, JsonEncoder,
		try_db_creation)
		@@ -1390,6 +1393,21 @@ class NetworkCandidateGraph(CandidateGraph):
		# for the sensor calls.
		self._setup_dem()

		@contextmanager
		def session_scope(self):
		"""
		Provide a transactional scope around a series of operations.
		"""
		session = self.Session()
		try:
		yield session
		session.commit()
		except:
		session.rollback()
		raise
		finally:
		session.close()

		def _setup_dem(self):
		spatial = self.config['spatial']
		dem = spatial.get('dem', False)
		@@ -1416,21 +1434,6 @@ class NetworkCandidateGraph(CandidateGraph):
		db=0)
		self.processing_queue = conf['processing_queue']

		@contextmanager
		def session_scope(self):
		"""
		Provide a transactional scope around a series of operations.
		"""
		session = self.Session()
		try:
		yield session
		session.commit()
		except:
		session.rollback()
		raise
		finally:
		session.close()

		def empty_queues(self):
		"""
		Delete all messages from the redis queue. This a convenience method.
		@@ -1661,87 +1664,54 @@ class NetworkCandidateGraph(CandidateGraph):
		if msg['success'] == True:
		return

		def to_isis(self, path, flistpath=None,sql = """
		SELECT measures."pointid",
		points."pointType",
		points."apriori",
		points."adjusted",
		points."pointIgnore",
		measures."id",
		measures."serialnumber",
		measures."sample",
		measures."line",
		measures."measureType",
		measures."imageid",
		measures."measureIgnore",
		measures."measureJigsawRejected",
		measures."aprioriline",
		measures."apriorisample"
		FROM measures
		INNER JOIN points ON measures."pointid" = points."id"
		WHERE
		points."pointIgnore" = False AND
		measures."measureIgnore" = FALSE AND
		measures."measureJigsawRejected" = FALSE AND
		measures."imageid" NOT IN
		(SELECT measures."imageid"
		FROM measures
		INNER JOIN points ON measures."pointid" = points."id"
		WHERE measures."measureIgnore" = False and measures."measureJigsawRejected" = False AND points."pointIgnore" = False
		GROUP BY measures."imageid"
		HAVING COUNT(DISTINCT measures."pointid") < 3)
		ORDER BY measures."pointid", measures."id";
		"""):
		"""
		Given a set of points/measures in an autocnet database, generate an ISIS
		compliant control network.
		def to_isis(self,
		path,
		flistpath=None,
		latsigma=10,
		lonsigma=10,
		radsigma=15,
		**db_kwargs):
		"""
		Write a NetworkCandidateGraph to an ISIS control network

		Parameters
		----------
		path : str
		The full path to the output network.
		Outpath to write the control network

		flistpath : str
		(Optional) the path to the output filelist. By default
		the outout filelist path is genrated programatically
		as the provided path with the extension replaced with .lis.
		For example, out.net would have an associated out.lis file.
		flishpath : str
		Outpath to write the associated file list. If None (default),
		the file list is written alongside the control network

		sql : str
		The sql query to execute in the database.
		latsigma : int/float
		The estimated sigma (error) in the latitude direction

		"""
		lonsigma : int/float
		The estimated sigma (error) in the longitude direction

		df = pd.read_sql(sql, self.engine)
		radsigma : int/float
		The estimated sigma (error) in the radius direction

		# measures.id DB column was read in to ensure the proper ordering of DF
		# so the correct measure is written as reference
		del df['id']
		df.rename(columns = {'pointid': 'id'}, inplace=True)
		radius : int/float
		The body semimajor radius

		#create columns in the dataframe; zeros ensure plio (/protobuf) will
		#ignore unless populated with alternate values
		df['aprioriX'] = 0
		df['aprioriY'] = 0
		df['aprioriZ'] = 0
		df['adjustedX'] = 0
		df['adjustedY'] = 0
		df['adjustedZ'] = 0
		db_kwargs : dict
		Kwargs that are passed to the io.db.controlnetwork.db_to_df function

		#only populate the new columns for ground points. Otherwise, isis will
		#recalculate the control point lat/lon from control measures which where
		#"massaged" by the phase and template matcher.
		for i, row in df.iterrows():
		if row['pointType'] == 3 or row['pointType'] == 4:
		apriori_geom = swkb.loads(row['apriori'], hex=True)
		row['aprioriX'] = apriori_geom.x
		row['aprioriY'] = apriori_geom.y
		row['aprioriZ'] = apriori_geom.z
		adjusted_geom = swkb.loads(row['adjusted'], hex=True)
		row['adjustedX'] = adjusted_geom.x
		row['adjustedY'] = adjusted_geom.y
		row['adjustedZ'] = adjusted_geom.z
		df.iloc[i] = row
		Returns
		-------
		None

		"""
		# Read the cnet from the db
		df = io_controlnetwork.db_to_df(self.engine, **db_kwargs)

		# Add the covariance matrices to ground measures
		df = control.compute_covariance(df,
		latsigma,
		lonsigma,
		radsigma,
		self.config['spatial']['semimajor_rad'])

		if flistpath is None:
		flistpath = os.path.splitext(path)[0] + '.lis'
		@@ -1753,6 +1723,10 @@ ORDER BY measures."pointid", measures."id";
		if f not in fpaths:
		warnings.warn(f'{f} in candidate graph but not in output network.')

		# Remap the df columns back to ISIS
		df.rename(columns={'pointtype':'pointType',
		'measuretype':'measureType'},
		inplace=True)
		cnet.to_isis(df, path, targetname=target)
		cnet.write_filelist(fpaths, path=flistpath)

		@@ -1828,7 +1802,6 @@ ORDER BY measures."pointid", measures."id";
		obj = cls.from_database()
		# Execute the computation to compute overlapping geometries
		obj._execute_sql(compute_overlaps_sql)

		return obj

		def copy_images(self, newdir):
		@@ -1857,7 +1830,6 @@ ORDER BY measures."pointid", measures."id";
		else:
		continue


		def add_from_remote_database(self, source_db_config, path, query_string='SELECT * FROM public.images LIMIT 10'):
		"""
		This is a constructor that takes an existing database containing images and sensors,

autocnet/graph/tests/test_network_graph.py

+11 −0

Original line number	Diff line number	Diff line
		import os
		import pytest

		import pandas as pd
		@@ -46,3 +47,13 @@ def test_place_points_from_cnet(session, cnet, image_data, expected_npoints, ncg
		assert len(resp.all()) == expected_npoints
		assert len(resp.all()) == cnet.shape[0]
		session.close()

		def test_to_isis(db_controlnetwork, ncg, node_a, node_b, tmpdir):
		ncg.add_edge(0,1)
		ncg.nodes[0]['data'] = node_a
		ncg.nodes[1]['data'] = node_b

		outpath = tmpdir.join('outnet.net')
		ncg.to_isis(outpath)

		assert os.path.exists(outpath)

autocnet/io/db/controlnetwork.py

0 → 100644

+87 −0

Original line number	Diff line number	Diff line
		import pandas as pd
		import shapely.wkb as swkb


		def db_to_df(engine, sql = """
		SELECT measures."pointid",
		points."pointType",
		points."apriori",
		points."adjusted",
		points."pointIgnore",
		measures."id",
		measures."serialnumber",
		measures."sample",
		measures."line",
		measures."measureType",
		measures."imageid",
		measures."measureIgnore",
		measures."measureJigsawRejected",
		measures."aprioriline",
		measures."apriorisample"
		FROM measures
		INNER JOIN points ON measures."pointid" = points."id"
		WHERE
		points."pointIgnore" = False AND
		measures."measureIgnore" = FALSE AND
		measures."measureJigsawRejected" = FALSE AND
		measures."imageid" NOT IN
		(SELECT measures."imageid"
		FROM measures
		INNER JOIN points ON measures."pointid" = points."id"
		WHERE measures."measureIgnore" = False and measures."measureJigsawRejected" = False AND points."pointIgnore" = False
		GROUP BY measures."imageid"
		HAVING COUNT(DISTINCT measures."pointid") < 3)
		ORDER BY measures."pointid", measures."id";
		"""):
		"""
		Given a set of points/measures in an autocnet database, generate an ISIS
		compliant control network.
		Parameters
		----------
		path : str
		The full path to the output network.
		flistpath : str
		(Optional) the path to the output filelist. By default
		the outout filelist path is genrated programatically
		as the provided path with the extension replaced with .lis.
		For example, out.net would have an associated out.lis file.
		sql : str
		The sql query to execute in the database.
		"""
		df = pd.read_sql(sql, engine)

		# measures.id DB column was read in to ensure the proper ordering of DF
		# so the correct measure is written as reference
		del df['id']
		df.rename(columns = {'pointid': 'id',
		'pointType': 'pointtype',
		'measureType': 'measuretype'}, inplace=True)

		#create columns in the dataframe; zeros ensure plio (/protobuf) will
		#ignore unless populated with alternate values
		df['aprioriX'] = 0
		df['aprioriY'] = 0
		df['aprioriZ'] = 0
		df['adjustedX'] = 0
		df['adjustedY'] = 0
		df['adjustedZ'] = 0
		df['aprioriCovar'] = [[] for _ in range(len(df))]

		#only populate the new columns for ground points. Otherwise, isis will
		#recalculate the control point lat/lon from control measures which where
		#"massaged" by the phase and template matcher.
		for i, row in df.iterrows():
		if row['pointtype'] == 3 or row['pointtype'] == 4:
		if row['apriori']:
		apriori_geom = swkb.loads(row['apriori'], hex=True)
		row['aprioriX'] = apriori_geom.x
		row['aprioriY'] = apriori_geom.y
		row['aprioriZ'] = apriori_geom.z
		if row['adjusted']:
		adjusted_geom = swkb.loads(row['adjusted'], hex=True)
		row['adjustedX'] = adjusted_geom.x
		row['adjustedY'] = adjusted_geom.y
		row['adjustedZ'] = adjusted_geom.z
		df[i] = row

		return df
		No newline at end of file