Unverified Commit db398139 authored by jlaura's avatar jlaura Committed by GitHub
Browse files

Adds arbitrary weights to the edges for use in then finding islands (#486) 

* Adds arbitrary weights to the edges for use in then finding islands

* edge filter now maintains orphaned nodes

* Forces PVL < 1.0

* Build needs PVL restriction too

* Finishes refactor of weight to be an attribute on edges

* removes code that isn't used/tested
parent 5fdb5d0f

autocnet/graph/edge.py

+39 −7
Original line number Diff line number Diff line 
from collections import defaultdict, MutableMapping, Counter

from functools import wraps, singledispatch

import json

import warnings

from collections import defaultdict, MutableMapping, Counter



import numpy as np

import pandas as pd
@@ -53,13 +54,11 @@ class Edge(dict, MutableMapping): 
        self['fundamental_matrix'] = None

        self.subpixel_matches = pd.DataFrame()

        self._matches = pd.DataFrame()

        self['weights'] = {}



        self['source_mbr'] = None

        self['destin_mbr'] = None

        self['overlap_latlon_coords'] = None





    def __repr__(self):

        return """

        Source Image Index: {}
@@ -67,11 +66,20 @@ class Edge(dict, MutableMapping): 
        Available Masks: {}

        """.format(self.source, self.destination, self.masks)





    def __eq__(self, other):

        return utils.compare_dicts(self.__dict__, other.__dict__) *\

               utils.compare_dicts(self, other)



    @property

    def weights(self):

        if not hasattr(self, '_weights'):

            self._weights = {}

        return self._weights



    @weights.setter

    def weights(self, kv):

        key, value = kv

        self.weights[key] = value



    @property

    def masks(self):
@@ -611,8 +619,8 @@ class Edge(dict, MutableMapping): 


        overlapinfo = cg.two_poly_overlap(poly1, poly2)



        self['weights']['overlap_area'] = overlapinfo[1]

        self['weights']['overlap_percn'] = overlapinfo[0]

        self.weights = ('overlap_area', overlapinfo[1])

        self.weights = ('overlap_percn', overlapinfo[0])



    def coverage(self, clean_keys = []):

        """
@@ -749,6 +757,29 @@ class NetworkEdge(Edge): 
            session.expunge_all()

        return res



    @property

    def weights(self):

        with self.parent.session_scope() as session:

            res = session.query(Edges.weights).\

                                            filter(Edges.source == self.source['node_id']).\

                                            filter(Edges.destination == self.destination['node_id']).\

                                            one()[0]

        self._weights = json.loads(res)

        return self._weights



    @weights.setter

    def weights(self, kv):

        key, value = kv

        with self.parent.session_scope() as session:

            res = session.query(Edges).\

                                            filter(Edges.source == self.source['node_id']).\

                                            filter(Edges.destination == self.destination['node_id']).\

                                            one()

            weights = json.loads(res.weights)

            weights[key] = value



            res.weights = json.dumps(weights)



    @property

    def parent(self):

        return getattr(self, '_parent', None)
@@ -975,6 +1006,7 @@ class NetworkEdge(Edge): 
    def measures(self):

        with self.parent.session_scope() as session:

            res = session.query(Measures).filter(sqlalchemy.or_(Measures.imageid == self.source['node_id'], Measures.imageid == self.destination['node_id'])).all()

            session.expunge_all()

        return res



    def network_to_matches(self, ignore_point=False, ignore_measure=False, rejected_jigsaw=False):

autocnet/graph/network.py

+48 −8
Original line number Diff line number Diff line
@@ -941,9 +941,11 @@ class CandidateGraph(nx.Graph): 
        : Object

          A networkX graph object

        """

        edges = [(u, v) for u, v, edge in self.edges.data(

        edges_to_remove = [(u, v) for u, v, edge in self.edges.data(

                            'data') if func(edge, *args, **kwargs)]

        return self.create_edge_subgraph(edges)

        subgraph = nx.create_empty_copy(self)

        subgraph.add_edges_from(edges_to_remove)

        return subgraph



    def compute_cliques(self, node_id=None):  # pragma: no cover

        """
@@ -1423,6 +1425,32 @@ class NetworkCandidateGraph(CandidateGraph): 
        # Attempt to create the database (if it does not exist)

        try_db_creation(self.engine, self.config)



    def _setup_edges(self):

        with self.session_scope() as session:

            res = session.query(Edges).all()

            

            edges = []

            for e in res:

                s = e.source

                d = e.destination

                if s > d:

                    s,d = d,s

                edges.append((s,d))

            

            to_add = []

            for e in self.edges:

                s = e[0]

                d = e[1]

                if s > d:

                    s,d = d,s

                edgeid = (s,d)

                if edgeid not in edges:

                    to_add.append(Edges(source=edgeid[0],

                                        destination=edgeid[1],

                                        weights=json.dumps({})))

            session.add_all(to_add)

            session.commit()

            print(len(to_add))

    def _setup_queues(self):

        """

        Setup a 2 queue redis connection for pushing and pulling work/results
@@ -1463,6 +1491,7 @@ class NetworkCandidateGraph(CandidateGraph): 
        """

        Push messages to the redis queue for objects e.g., Nodes and Edges

        """



        for job_counter, elem in enumerate(onobj.data('data')):

            if getattr(elem[-1], 'ignore', False):

                continue
@@ -1539,8 +1568,11 @@ class NetworkCandidateGraph(CandidateGraph): 
        Parameters

        ----------



        function : string

                   The function to apply

        function : string / obj

                   The function to apply. This can be either the full, importable path from

                   this library or an arbitrary function that will be serialized. If the arbitrary

                   function requires imports external to this library, those imports must be made

                   within the function scope.



        on : str

             {'edge', 'edges', 'e', 0} for an edge
@@ -1605,15 +1637,20 @@ class NetworkCandidateGraph(CandidateGraph): 
            on='overlaps', distribute_points_kwargs=distribute_points_kwargs)

        """



        job_counter = self.queue_length()

        job_counter = self.queue_length



        if not reapply:

            # Determine which obj will be called

            onobj = self.apply_iterable_options[on]

            res = []



            # This method support arbitrary functions. The name needs to be a string for the log name.

            if not isinstance(function, (str, bytes)):

                raise TypeError('Function argument must be a string or bytes object.')

                function_name = function.__name__

            else:

                function_name = function



            # Dispatch to either the database object message generator or the autocnet object message generator

            if isinstance(onobj, DeclarativeMeta):

                job_counter = self._push_row_messages(onobj, on, function, walltime, filters, query_string, args, kwargs)

            else:
@@ -1641,7 +1678,7 @@ class NetworkCandidateGraph(CandidateGraph): 
                     mem_per_cpu=self.config['cluster']['processing_memory'],

                     time=walltime,

                     partition=self.config['cluster']['queue'],

                     output=self.config['cluster']['cluster_log_dir']+f'/autocnet.{function}-%j')

                     output=self.config['cluster']['cluster_log_dir']+f'/autocnet.{function_name}-%j')

        submitter.submit(array='1-{}%{}'.format(job_counter,arraychunk), chunksize=chunksize)

        return job_counter
@@ -1966,6 +2003,9 @@ class NetworkCandidateGraph(CandidateGraph): 
        # Add nodes that do not overlap any images

        self.__init__(adjacency, node_id_map=adjacency_lookup)

        

        # Setup the edges

        self._setup_edges()



    @staticmethod

    def clear_db(tables=None):

        """

autocnet/graph/tests/test_edge.py

+10 −3
Original line number Diff line number Diff line
@@ -29,7 +29,6 @@ class TestEdge(unittest.TestCase): 


    def test_edge_overlap(self):

        e = edge.Edge()

        e.weight = {}

        source = Mock(spec = node.Node)

        destination = Mock(spec = node.Node)

        e.destination = destination
@@ -49,8 +48,16 @@ class TestEdge(unittest.TestCase): 
        destination.geodata.footprint = poly2



        e.overlap()

        self.assertEqual(e['weights']['overlap_area'], 400)

        self.assertAlmostEqual(e['weights']['overlap_percn'], 14.285714285)

        self.assertEqual(e.weights['overlap_area'], 400)

        self.assertAlmostEqual(e.weights['overlap_percn'], 14.285714285)



    def test_weight(self):

        e = edge.Edge()

        assert e.weights == {}

        e.weights = ('foo', 1)

        assert e.weights['foo']  == 1

        e.weights = ('foo', 2)

        assert e.weights['foo']  == 2



    def test_coverage(self):

        adjacency = get_path('two_image_adjacency.json')

autocnet/graph/tests/test_network.py

+0 −2
Original line number Diff line number Diff line
@@ -213,13 +213,11 @@ def test_get_matches(candidategraph): 
def test_island_nodes(disconnected_graph):

    assert len(list(disconnected_graph.island_nodes())) == 1





def test_triangular_cycles(graph):

    cycles = graph.compute_triangular_cycles()

    # Node order is variable, length is not

    assert len(cycles) == 1





def test_connected_subgraphs(graph, disconnected_graph):

    # Calls all return generators, cast to list for positional comparison

    subgraph_list = list(disconnected_graph.connected_subgraphs())

autocnet/io/db/model.py

+1 −0
Original line number Diff line number Diff line
@@ -128,6 +128,7 @@ class Edges(BaseMixin, Base): 
    fundamental = Column(ArrayType())

    ignore = Column(Boolean, default=False)

    masks = Column(Json())

    weights = Column(Json())



class Costs(BaseMixin, Base):

    __tablename__ = 'costs'