Merge branch 'sqlString' into 'dev' (d7aefd46) · Commits · aflab / astrogeology / Autocnet

autocnet/graph/network.py

+6 −10

Original line number	Diff line number	Diff line
		@@ -33,6 +33,7 @@ from plio.io.io_gdal import GeoDataset
		from plio.io.isis_serial_number import generate_serial_number
		from plio.io import io_controlnetwork as cnet

		from .. import sql

		from plurmy import Slurm

		@@ -53,7 +54,6 @@ from autocnet.matcher import subpixel
		from autocnet.matcher import cross_instrument_matcher as cim
		from autocnet.vis.graph_view import plot_graph, cluster_plot
		from autocnet.control import control
		from autocnet.spatial.overlap import compute_overlaps_sql
		from autocnet.spatial.isis import point_info
		from autocnet.spatial.surface import GdalDem, EllipsoidDem
		from autocnet.transformation.spatial import reproject, og2oc
		@@ -2080,7 +2080,7 @@ class NetworkCandidateGraph(CandidateGraph):

		self.from_database()
		# Execute the computation to compute overlapping geometries
		self._execute_sql(compute_overlaps_sql)
		self._execute_sql(sql.compute_overlaps_sql)

		def add_image(self, img_path):
		"""
		@@ -2128,7 +2128,7 @@ class NetworkCandidateGraph(CandidateGraph):
		else:
		continue

		def add_from_remote_database(self, source_db_config, path=None, query_string='SELECT * FROM public.images LIMIT 10'):
		def add_from_remote_database(self, source_db_config, path=None, query_string=sql.select_ten_pub_image):
		"""
		This is a constructor that takes an existing database containing images and sensors,
		copies the selected rows into the project specified in the autocnet_config variable,
		@@ -2199,9 +2199,9 @@ class NetworkCandidateGraph(CandidateGraph):
		if path:
		self.copy_images(path)
		self.from_database()
		self._execute_sql(compute_overlaps_sql)
		self._execute_sql(sql.compute_overlaps_sql)

		def from_database(self, query_string='SELECT * FROM public.images'):
		def from_database(self, query_string=sql.select_pub_image):
		"""
		This is a constructor that takes the results from an arbitrary query string,
		uses those as a subquery into a standard polygon overlap query and
		@@ -2234,11 +2234,7 @@ class NetworkCandidateGraph(CandidateGraph):
		SELECT * FROM Images WHERE (split_part(path, '/', 6) ~ 'P[0-9]+_.+') = True
		"""

		composite_query = '''WITH i as ({}) SELECT i1.id
		as i1_id,i1.path as i1_path, i2.id as i2_id, i2.path as i2_path
		FROM i as i1, i as i2
		WHERE ST_INTERSECTS(i1.geom, i2.geom) = TRUE
		AND i1.id < i2.id'''.format(query_string)
		composite_query = sql.from_database_composite.format(formatInput=sql.select_pub_image)

		with self.session_scope() as session:
		res = session.execute(composite_query)

autocnet/graph/tests/test_network.py

+2 −1

Original line number	Diff line number	Diff line
		@@ -19,6 +19,7 @@ from autocnet.examples import get_path
		from .. import network
		from .. import edge
		from .. import node
		import warnings

		sys.path.insert(0, os.path.abspath('..'))

autocnet/io/db/controlnetwork.py

+2 −33

Original line number	Diff line number	Diff line
		@@ -7,40 +7,9 @@ import shapely.wkb as swkb
		from plio.io import io_controlnetwork as cnet
		from autocnet.io.db.model import Measures
		from autocnet.spatial.isis import isis2np_types
		from ... import sql

		def db_to_df(engine, sql = """
		SELECT measures."pointid",
		points."pointType",
		points."apriori",
		points."adjusted",
		points."pointIgnore",
		points."referenceIndex",
		points."identifier",
		measures."id",
		measures."serialnumber",
		measures."sample",
		measures."line",
		measures."measureType",
		measures."imageid",
		measures."measureIgnore",
		measures."measureJigsawRejected",
		measures."aprioriline",
		measures."apriorisample"
		FROM measures
		INNER JOIN points ON measures."pointid" = points."id"
		WHERE
		points."pointIgnore" = False AND
		measures."measureIgnore" = FALSE AND
		measures."measureJigsawRejected" = FALSE AND
		measures."imageid" NOT IN
		(SELECT measures."imageid"
		FROM measures
		INNER JOIN points ON measures."pointid" = points."id"
		WHERE measures."measureIgnore" = False and measures."measureJigsawRejected" = False AND points."pointIgnore" = False
		GROUP BY measures."imageid"
		HAVING COUNT(DISTINCT measures."pointid") < 3)
		ORDER BY measures."pointid", measures."id";
		"""):
		def db_to_df(engine, sql = sql.db_to_df_sql_string):
		"""
		Given a set of points/measures in an autocnet database, generate an ISIS
		compliant control network.

autocnet/io/db/triggers.py

+24 −199

Original line number	Diff line number	Diff line
		from sqlalchemy.schema import DDL
		from ... import sql

		valid_geom_function = DDL("""
		CREATE OR REPLACE FUNCTION validate_geom()
		RETURNS trigger AS
		$BODY$
		BEGIN
		NEW.geom = ST_MAKEVALID(NEW.geom);
		RETURN NEW;
		EXCEPTION WHEN OTHERS THEN
		NEW.ignore = true;
		RETURN NEW;
		END;
		$BODY$

		LANGUAGE plpgsql VOLATILE -- Says the function is implemented in the plpgsql language; VOLATILE says the function has side effects.
		COST 100; -- Estimated execution cost of the function.
		""")

		valid_geom_trigger = DDL("""
		CREATE TRIGGER image_inserted
		BEFORE INSERT OR UPDATE
		ON images
		FOR EACH ROW
		EXECUTE PROCEDURE validate_geom();
		""")

		valid_point_function = DDL("""
		CREATE OR REPLACE FUNCTION validate_points()
		RETURNS trigger AS
		$BODY$
		BEGIN
		IF (SELECT COUNT(*)
		FROM MEASURES
		WHERE pointid = NEW.pointid AND "measureIgnore" = False) < 2
		THEN
		UPDATE points
		SET "pointIgnore" = True
		WHERE points.id = NEW.pointid;
		ELSE
		UPDATE points
		SET "pointIgnore" = False
		WHERE points.id = NEW.pointid;
		END IF;

		RETURN NEW;
		END;
		$BODY$

		LANGUAGE plpgsql VOLATILE -- Says the function is implemented in the plpgsql language; VOLATILE says the function has side effects.
		COST 100; -- Estimated execution cost of the function.
		""")

		valid_point_trigger = DDL("""
		CREATE TRIGGER active_measure_changes
		AFTER UPDATE
		ON measures
		FOR EACH ROW
		EXECUTE PROCEDURE validate_points();
		""")

		ignore_image_function = DDL("""
		CREATE OR REPLACE FUNCTION ignore_image()
		RETURNS trigger AS
		$BODY$
		BEGIN
		IF NEW.ignore
		THEN
		UPDATE measures
		SET "measureIgnore" = True
		WHERE measures.serialnumber = NEW.serial;
		END IF;

		RETURN NEW;
		END;
		$BODY$

		LANGUAGE plpgsql VOLATILE -- Says the function is implemented in the plpgsql language; VOLATILE says the function has side effects.
		COST 100; -- Estimated execution cost of the function.
		""")

		ignore_image_trigger = DDL("""
		CREATE TRIGGER image_ignored
		AFTER UPDATE
		ON images
		FOR EACH ROW
		EXECUTE PROCEDURE ignore_image();
		""")
		valid_geom_function = DDL(sql.valid_geom_func_str)

		valid_geom_trigger = DDL(sql.valid_geom_trig_str)

		valid_point_function = DDL(sql.valid_point_func_str)

		valid_point_trigger = DDL(sql.valid_point_trig_str)

		ignore_image_function = DDL(sql.ignore_image_fun_str)

		ignore_image_trigger = DDL(sql.ignore_image_trig_str)


		# several funcs and an operator needed to get json diff working.
		jsonb_delete_func = DDL("""
		SET search_path = 'public';

		CREATE OR REPLACE FUNCTION jsonb_delete_left(a jsonb, b text)
		RETURNS jsonb AS
		$BODY$
		SELECT COALESCE(
		(
		SELECT ('{' \|\| string_agg(to_json(key) \|\| ':' \|\| value, ',') \|\| '}')
		FROM jsonb_each(a)
		WHERE key <> b
		)
		, '{}')::jsonb;
		$BODY$
		LANGUAGE sql IMMUTABLE STRICT;
		COMMENT ON FUNCTION jsonb_delete_left(jsonb, text) IS 'delete key in second argument from first argument';

		CREATE OPERATOR - ( PROCEDURE = jsonb_delete_left, LEFTARG = jsonb, RIGHTARG = text);
		COMMENT ON OPERATOR - (jsonb, text) IS 'delete key from left operand';

		--

		CREATE OR REPLACE FUNCTION jsonb_delete_left(a jsonb, b text[])
		RETURNS jsonb AS
		$BODY$
		SELECT COALESCE(
		(
		SELECT ('{' \|\| string_agg(to_json(key) \|\| ':' \|\| value, ',') \|\| '}')
		FROM jsonb_each(a)
		WHERE key <> ALL(b)
		)
		, '{}')::jsonb;
		$BODY$
		LANGUAGE sql IMMUTABLE STRICT;
		COMMENT ON FUNCTION jsonb_delete_left(jsonb, text[]) IS 'delete keys in second argument from first argument';

		CREATE OPERATOR - ( PROCEDURE = jsonb_delete_left, LEFTARG = jsonb, RIGHTARG = text[]);
		COMMENT ON OPERATOR - (jsonb, text[]) IS 'delete keys from left operand';

		--

		CREATE OR REPLACE FUNCTION jsonb_delete_left(a jsonb, b jsonb)
		RETURNS jsonb AS
		$BODY$
		SELECT COALESCE(
		(
		SELECT ('{' \|\| string_agg(to_json(key) \|\| ':' \|\| value, ',') \|\| '}')
		FROM jsonb_each(a)
		WHERE NOT ('{' \|\| to_json(key) \|\| ':' \|\| value \|\| '}')::jsonb <@ b
		)
		, '{}')::jsonb;
		$BODY$
		LANGUAGE sql IMMUTABLE STRICT;
		COMMENT ON FUNCTION jsonb_delete_left(jsonb, jsonb) IS 'delete matching pairs in second argument from first argument';

		CREATE OPERATOR - ( PROCEDURE = jsonb_delete_left, LEFTARG = jsonb, RIGHTARG = jsonb);
		COMMENT ON OPERATOR - (jsonb, jsonb) IS 'delete matching pairs from left operand';
		""")
		jsonb_delete_func = DDL(sql.json_delete_func_str)


		def generate_history_triggers(table):
		tablename = table.__tablename__

		history_update_function = DDL(f"""
		CREATE OR REPLACE FUNCTION {tablename}_history_update()
		RETURNS TRIGGER AS $$
		DECLARE
		js_new jsonb := row_to_json(NEW)::jsonb;
		js_old jsonb := row_to_json(OLD)::jsonb;
		BEGIN
		INSERT INTO {tablename}_history(fk, "eventTime", "executedBy", event, before, after)
		VALUES((js_old->>'id')::int, CURRENT_TIMESTAMP, SESSION_USER, 'update', js_old - js_new, js_new - js_old);
		RETURN NEW;
		END;

		$$ LANGUAGE plpgsql;
		""")

		history_insert_function = DDL(f"""
		CREATE OR REPLACE FUNCTION {tablename}_history_insert()
		RETURNS TRIGGER AS $$
		DECLARE
		js_new jsonb := row_to_json(NEW)::jsonb;
		BEGIN
		INSERT INTO {tablename}_history(fk, "eventTime", "executedBy", event, after)
		VALUES((js_new->>'id')::int, CURRENT_TIMESTAMP, SESSION_USER, 'insert', js_new);
		RETURN NEW;
		END;
		$$ LANGUAGE plpgsql;
		""")

		history_delete_function = DDL(f"""
		CREATE OR REPLACE FUNCTION {tablename}_history_delete()
		RETURNS TRIGGER AS $$
		DECLARE
		js_old jsonb := row_to_json(OLD)::jsonb;
		BEGIN
		INSERT INTO {tablename}_history(fk, "eventTime", "executedBy", event, before)
		VALUES((js_old->>'id')::int, CURRENT_TIMESTAMP, SESSION_USER, 'delete', js_old);
		RETURN NEW;
		END;
		$$ LANGUAGE plpgsql;
		""")

		history_insert_trigger = DDL(f"""
		CREATE TRIGGER {tablename}_history_insert AFTER INSERT ON {tablename}
		FOR EACH ROW EXECUTE PROCEDURE {tablename}_history_insert();
		""")

		history_delete_trigger = DDL(f"""
		CREATE TRIGGER {tablename}_history_delete AFTER DELETE ON {tablename}
		FOR EACH ROW EXECUTE PROCEDURE {tablename}_history_delete();
		""")

		history_update_trigger = DDL(f"""
		CREATE TRIGGER {tablename}_history_update AFTER UPDATE ON {tablename}
		FOR EACH ROW EXECUTE PROCEDURE {tablename}_history_update();
		""")
		history_update_function = DDL(sql.history_update_func_str.format(formatInput=tablename))

		history_insert_function = DDL(sql.history_insert_func_str.format(formatInput=tablename))

		history_delete_function = DDL(sql.history_delete_func_str.format(formatInput=tablename))

		history_insert_trigger = DDL(sql.history_insert_trig_str.format(formatInput=tablename))

		history_delete_trigger = DDL(sql.history_delete_trig_str.format(formatInput=tablename))

		history_update_trigger = DDL(sql.history_update_trig_str.format(formatInput=tablename))

		return history_update_function, history_insert_function, history_delete_function, history_insert_trigger, history_delete_trigger, history_update_trigger
		No newline at end of file

autocnet/spatial/overlap.py

+1 −21

Original line number	Diff line number	Diff line
		@@ -10,6 +10,7 @@ import shapely
		import sqlalchemy
		from plio.io.io_gdal import GeoDataset


		from autocnet.cg import cg as compgeom
		from autocnet.graph.node import NetworkNode
		from autocnet.io.db.model import Images, Measures, Overlay, Points, JsonEncoder
		@@ -21,27 +22,6 @@ from autocnet.transformation import roi
		from plurmy import Slurm
		import csmapi

		# SQL query to decompose pairwise overlaps
		compute_overlaps_sql = """
		WITH intersectiongeom AS
		(SELECT geom AS geom FROM ST_Dump((
		SELECT ST_Polygonize(the_geom) AS the_geom FROM (
		SELECT ST_Union(the_geom) AS the_geom FROM (
		SELECT ST_ExteriorRing((ST_DUMP(geom)).geom) AS the_geom
		FROM images WHERE images.geom IS NOT NULL) AS lines
		) AS noded_lines))),
		iid AS (
		SELECT images.id, intersectiongeom.geom AS geom
		FROM images, intersectiongeom
		WHERE images.geom is NOT NULL AND
		ST_INTERSECTS(intersectiongeom.geom, images.geom) AND
		ST_AREA(ST_INTERSECTION(intersectiongeom.geom, images.geom)) > 0.000001
		)
		INSERT INTO overlay(intersections, geom) SELECT row.intersections, row.geom FROM
		(SELECT iid.geom, array_agg(iid.id) AS intersections
		FROM iid GROUP BY iid.geom) AS row WHERE array_length(intersections, 1) > 1;
		"""

		# set up the logger file
		log = logging.getLogger(__name__)