Loading autocnet/graph/edge.py +2 −2 Original line number Diff line number Diff line Loading @@ -390,8 +390,8 @@ class Edge(dict, MutableMapping): raise AttributeError('This edge does not yet have any matches computed.') matches, mask = self.clean(clean_keys) domain = self.source.geodata.raster_size rs = self.source.geodata.raster_size domain = [0, 0, rs[0], rs[1]] # Massage the dataframe into the correct structure coords = self.source.get_keypoint_coordinates() merged = matches.merge(coords, left_on=['source_idx'], right_index=True) Loading autocnet/matcher/cpu_outlier_detector.py +73 −86 Original line number Diff line number Diff line Loading @@ -45,133 +45,116 @@ def distance_ratio(edge, matches, ratio=0.8, single=False): return mask def spatial_suppression(df, domain, min_radius=1.5, k=250, error_k=0.1): def spatial_suppression(df, bounds, xkey='lon', ykey='lat', k=60, error_k=0.05, nsteps=250): """ Spatial suppression using disc based method. Apply the spatial suppression algorithm over an arbitrary domain for all of the spatial data in the provided data frame. Attributes Parameters ---------- df : dataframe Input dataframe used for suppressing mask : series pandas boolean series df : object Pandas data frame with coordinates max_radius : float Maximum allowable point radius bounds : list In the form xmin, ymin, xmax, ymax min_radius : float The smallest allowable radius size xkey : str The column name for the x coordinates nvalid : int The number of valid points after suppression ykey : str The column name for the y coordinates k : int The number of points to be saved The desired number of points after suppression error_k : float [0,1] the acceptable error in k The percentage of allowable error in the domain [0,1] domain : tuple The (x,y) extent of the input domain nsteps : int The granularity of the search. This controls the number of buckets in the x and y dimension. More granular search adds processing time, but can result in a more accurate solution. Returns ------- mask : pd.Series Boolean suppression mask k : int The number of unsuppressed observations References ---------- [Gauglitz2011]_ mask : nd.array A boolean mask of the valid points len(result) : int The numer of valud points """ columns = df.columns for i in ['x', 'y', 'strength']: if i not in columns: raise ValueError('The dataframe is missing a {} column.'.format(i)) df = df.sort_values(by=['strength'], ascending=False).copy() # Compute the bounding area inside of which the suppression will be applied minx = min(bounds[0], bounds[2]) maxx = max(bounds[0], bounds[2]) miny = min(bounds[1], bounds[3]) maxy = max(bounds[1], bounds[3]) domain = (maxx-minx),(maxy-miny) min_radius = min(domain) / 20 max_radius = max(domain) mask = pd.Series(False, index=df.index) process = True if k > len(df): warnings.warn('Only {} valid points, but {} points requested'.format(len(df), k)) k = len(df) result = df.index process = False nsteps = max(domain) * 0.95 search_space = np.linspace(min_radius, max_radius, nsteps) cell_sizes = search_space / math.sqrt(2) min_idx = 0 max_idx = len(search_space) - 1 # Setup flags to watch for looping prev_min = None prev_max = None while process: # Setup to store results result = [] # Sort the dataframe (hard coded to ascending as lower strength (cost) is better) df = df.sort_values(by=['strength'], ascending=True).copy() df = df.reset_index(drop=True) mask = pd.Series(False, index=df.index) process = True while process: # Binary search mid_idx = int((min_idx + max_idx) / 2) if min_idx == mid_idx or mid_idx == max_idx: warnings.warn('Unable to optimally solve. Returning with {} points'.format(len(result))) warnings.warn('Unable to optimally solve.') process = False else: # Setup to store results result = [] # Get the current cell size and grid the domain cell_size = cell_sizes[mid_idx] n_x_cells = int(domain[0] / cell_size) n_y_cells = int(domain[1] / cell_size) grid = np.zeros((n_x_cells, n_y_cells), dtype=np.bool) n_x_cells = int(round(domain[0] / cell_size, 0)) - 1 n_y_cells = int(round(domain[1] / cell_size, 0)) - 1 if n_x_cells <= 0: n_x_cells = 1 if n_y_cells <= 0: n_y_cells = 1 grid = np.zeros((n_y_cells, n_x_cells), dtype=np.bool) # Assign all points to bins x_edges = np.linspace(0, domain[0], n_x_cells) y_edges = np.linspace(0, domain[1], n_y_cells) xbins = np.digitize(df['x'], bins=x_edges) ybins = np.digitize(df['y'], bins=y_edges) # Convert bins to cells xbins -= 1 ybins -= 1 pts = [] x_edges = np.linspace(minx, maxx, n_x_cells) y_edges = np.linspace(miny, maxy, n_y_cells) xbins = np.digitize(df[xkey], bins=x_edges) ybins = np.digitize(df[ykey], bins=y_edges) # Starting with the best point, start assigning points to grid cells for i, (idx, p) in enumerate(df.iterrows()): x_center = xbins[i] y_center = ybins[i] x_center = xbins[i] - 1 y_center = ybins[i] - 1 cell = grid[y_center, x_center] if cell == False: result.append(idx) pts.append((p[['x', 'y']])) if len(result) > k + k * error_k: # Too many points, break min_idx = mid_idx break y_min = y_center - int(round(cell_size, 0)) if y_min < 0: y_min = 0 x_min = x_center - int(round(cell_size, 0)) if x_min < 0: x_min = 0 # Set the cell to True grid[y_center, x_center] = True y_max = y_center + int(round(cell_size, 0)) if y_max > grid.shape[0]: y_max = grid.shape[0] # If everything is already 'covered' break from the list if grid.all() == False: continue x_max = x_center + int(round(cell_size, 0)) if x_max > grid.shape[1]: x_max = grid.shape[1] # Cover the necessary cells grid[y_min: y_max, x_min: x_max] = True # Check break conditions # Check to see if the algorithm is completed, or if the grid size needs to be larger or smaller if k - k * error_k <= len(result) <= k + k * error_k: # Success, in bounds process = False elif len(result) < k - k * error_k: # The radius is too large max_idx = mid_idx Loading @@ -181,10 +164,14 @@ def spatial_suppression(df, domain, min_radius=1.5, k=250, error_k=0.1): process = False if min_idx == max_idx: process = False mask = pd.Series(False, df.index) elif len(result) > k + k * error_k: # Too many points, break min_idx = mid_idx mask.loc[list(result)] = True return mask, k return mask, len(result) def self_neighbors(matches): Loading autocnet/matcher/tests/test_outlier_detector.py +17 −20 Original line number Diff line number Diff line Loading @@ -52,24 +52,21 @@ class TestSpatialSuppression(unittest.TestCase): y = seed.randint(0, 100, 100).astype(np.float32) strength = seed.rand(100) data = np.vstack((x, y, strength)).T self.df = pd.DataFrame(data, columns=['x', 'y', 'strength']) self.domain = (100,100) self.df = pd.DataFrame(data, columns=['lon', 'lat', 'strength']) self.domain = (0,0, 100, 100) def test_suppress_non_optimal(self): with warnings.catch_warnings(record=True) as w: def test_suppress(self): mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=25) self.assertEqual(len(w), 1) self.assertEqual(w[0].category, UserWarning) self.assertEqual(mask.sum(), 25) self.assertEqual(mask.sum(), 28) def test_suppress(self): mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=30) self.assertIn(mask.sum(), list(range(27, 35))) mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=30, error_k=.15) self.assertEqual(mask.sum(), 34) def test_suppress_non_optimal(self): with warnings.catch_warnings(record=True) as w: mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=101) mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=30) self.assertEqual(len(w), 1) self.assertEqual(k, 59) self.assertTrue(issubclass(w[0].category, UserWarning)) class testSuppressionRanges(unittest.TestCase): Loading @@ -79,21 +76,21 @@ class testSuppressionRanges(unittest.TestCase): cls.r = np.random.RandomState(12345) def test_min_max(self): df = pd.DataFrame(self.r.uniform(0,2,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (1.5,1.5), k = 1) df = pd.DataFrame(self.r.uniform(0,2,(500, 3)), columns=['lon', 'lat', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 1.5, 1.5), k = 1) self.assertEqual(len(df[mask]), 1) def test_point_overload(self): df = pd.DataFrame(self.r.uniform(0,15,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (15,15), k = 200) self.assertEqual(len(df[mask]), 69) df = pd.DataFrame(self.r.uniform(0,15,(500, 3)), columns=['lon', 'lat', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 15, 15), k = 200) self.assertEqual(len(df[mask]), 195) def test_small_distribution(self): df = pd.DataFrame(self.r.uniform(0,25,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (25,25), k = 25) self.assertEqual(len(df[mask]), 28) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 25, 25), k = 25, xkey='x', ykey='y') self.assertEqual(len(df[mask]), 25) def test_normal_distribution(self): df = pd.DataFrame(self.r.uniform(0,100,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (100,100), k = 15) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 100, 100), k = 15, xkey='x', ykey='y') self.assertEqual(len(df[mask]), 17) conda/meta.yaml +1 −1 Original line number Diff line number Diff line package: name: autocnet version: 0.2.0 version: 0.2.1 channels: - conda-forge Loading setup.py +1 −4 Original line number Diff line number Diff line Loading @@ -6,9 +6,6 @@ from autocnet.examples import available with open('README.rst', 'r') as f: long_description = f.read() VERSION = autocnet.__version__ def setup_package(): examples = set() for i in available(): Loading @@ -23,7 +20,7 @@ def setup_package(): setup( name = "autocnet", version = VERSION, version = '0.2.1', author = "Jay Laura", author_email = "jlaura@usgs.gov", description = ("I/O API to support planetary data formats."), Loading Loading
autocnet/graph/edge.py +2 −2 Original line number Diff line number Diff line Loading @@ -390,8 +390,8 @@ class Edge(dict, MutableMapping): raise AttributeError('This edge does not yet have any matches computed.') matches, mask = self.clean(clean_keys) domain = self.source.geodata.raster_size rs = self.source.geodata.raster_size domain = [0, 0, rs[0], rs[1]] # Massage the dataframe into the correct structure coords = self.source.get_keypoint_coordinates() merged = matches.merge(coords, left_on=['source_idx'], right_index=True) Loading
autocnet/matcher/cpu_outlier_detector.py +73 −86 Original line number Diff line number Diff line Loading @@ -45,133 +45,116 @@ def distance_ratio(edge, matches, ratio=0.8, single=False): return mask def spatial_suppression(df, domain, min_radius=1.5, k=250, error_k=0.1): def spatial_suppression(df, bounds, xkey='lon', ykey='lat', k=60, error_k=0.05, nsteps=250): """ Spatial suppression using disc based method. Apply the spatial suppression algorithm over an arbitrary domain for all of the spatial data in the provided data frame. Attributes Parameters ---------- df : dataframe Input dataframe used for suppressing mask : series pandas boolean series df : object Pandas data frame with coordinates max_radius : float Maximum allowable point radius bounds : list In the form xmin, ymin, xmax, ymax min_radius : float The smallest allowable radius size xkey : str The column name for the x coordinates nvalid : int The number of valid points after suppression ykey : str The column name for the y coordinates k : int The number of points to be saved The desired number of points after suppression error_k : float [0,1] the acceptable error in k The percentage of allowable error in the domain [0,1] domain : tuple The (x,y) extent of the input domain nsteps : int The granularity of the search. This controls the number of buckets in the x and y dimension. More granular search adds processing time, but can result in a more accurate solution. Returns ------- mask : pd.Series Boolean suppression mask k : int The number of unsuppressed observations References ---------- [Gauglitz2011]_ mask : nd.array A boolean mask of the valid points len(result) : int The numer of valud points """ columns = df.columns for i in ['x', 'y', 'strength']: if i not in columns: raise ValueError('The dataframe is missing a {} column.'.format(i)) df = df.sort_values(by=['strength'], ascending=False).copy() # Compute the bounding area inside of which the suppression will be applied minx = min(bounds[0], bounds[2]) maxx = max(bounds[0], bounds[2]) miny = min(bounds[1], bounds[3]) maxy = max(bounds[1], bounds[3]) domain = (maxx-minx),(maxy-miny) min_radius = min(domain) / 20 max_radius = max(domain) mask = pd.Series(False, index=df.index) process = True if k > len(df): warnings.warn('Only {} valid points, but {} points requested'.format(len(df), k)) k = len(df) result = df.index process = False nsteps = max(domain) * 0.95 search_space = np.linspace(min_radius, max_radius, nsteps) cell_sizes = search_space / math.sqrt(2) min_idx = 0 max_idx = len(search_space) - 1 # Setup flags to watch for looping prev_min = None prev_max = None while process: # Setup to store results result = [] # Sort the dataframe (hard coded to ascending as lower strength (cost) is better) df = df.sort_values(by=['strength'], ascending=True).copy() df = df.reset_index(drop=True) mask = pd.Series(False, index=df.index) process = True while process: # Binary search mid_idx = int((min_idx + max_idx) / 2) if min_idx == mid_idx or mid_idx == max_idx: warnings.warn('Unable to optimally solve. Returning with {} points'.format(len(result))) warnings.warn('Unable to optimally solve.') process = False else: # Setup to store results result = [] # Get the current cell size and grid the domain cell_size = cell_sizes[mid_idx] n_x_cells = int(domain[0] / cell_size) n_y_cells = int(domain[1] / cell_size) grid = np.zeros((n_x_cells, n_y_cells), dtype=np.bool) n_x_cells = int(round(domain[0] / cell_size, 0)) - 1 n_y_cells = int(round(domain[1] / cell_size, 0)) - 1 if n_x_cells <= 0: n_x_cells = 1 if n_y_cells <= 0: n_y_cells = 1 grid = np.zeros((n_y_cells, n_x_cells), dtype=np.bool) # Assign all points to bins x_edges = np.linspace(0, domain[0], n_x_cells) y_edges = np.linspace(0, domain[1], n_y_cells) xbins = np.digitize(df['x'], bins=x_edges) ybins = np.digitize(df['y'], bins=y_edges) # Convert bins to cells xbins -= 1 ybins -= 1 pts = [] x_edges = np.linspace(minx, maxx, n_x_cells) y_edges = np.linspace(miny, maxy, n_y_cells) xbins = np.digitize(df[xkey], bins=x_edges) ybins = np.digitize(df[ykey], bins=y_edges) # Starting with the best point, start assigning points to grid cells for i, (idx, p) in enumerate(df.iterrows()): x_center = xbins[i] y_center = ybins[i] x_center = xbins[i] - 1 y_center = ybins[i] - 1 cell = grid[y_center, x_center] if cell == False: result.append(idx) pts.append((p[['x', 'y']])) if len(result) > k + k * error_k: # Too many points, break min_idx = mid_idx break y_min = y_center - int(round(cell_size, 0)) if y_min < 0: y_min = 0 x_min = x_center - int(round(cell_size, 0)) if x_min < 0: x_min = 0 # Set the cell to True grid[y_center, x_center] = True y_max = y_center + int(round(cell_size, 0)) if y_max > grid.shape[0]: y_max = grid.shape[0] # If everything is already 'covered' break from the list if grid.all() == False: continue x_max = x_center + int(round(cell_size, 0)) if x_max > grid.shape[1]: x_max = grid.shape[1] # Cover the necessary cells grid[y_min: y_max, x_min: x_max] = True # Check break conditions # Check to see if the algorithm is completed, or if the grid size needs to be larger or smaller if k - k * error_k <= len(result) <= k + k * error_k: # Success, in bounds process = False elif len(result) < k - k * error_k: # The radius is too large max_idx = mid_idx Loading @@ -181,10 +164,14 @@ def spatial_suppression(df, domain, min_radius=1.5, k=250, error_k=0.1): process = False if min_idx == max_idx: process = False mask = pd.Series(False, df.index) elif len(result) > k + k * error_k: # Too many points, break min_idx = mid_idx mask.loc[list(result)] = True return mask, k return mask, len(result) def self_neighbors(matches): Loading
autocnet/matcher/tests/test_outlier_detector.py +17 −20 Original line number Diff line number Diff line Loading @@ -52,24 +52,21 @@ class TestSpatialSuppression(unittest.TestCase): y = seed.randint(0, 100, 100).astype(np.float32) strength = seed.rand(100) data = np.vstack((x, y, strength)).T self.df = pd.DataFrame(data, columns=['x', 'y', 'strength']) self.domain = (100,100) self.df = pd.DataFrame(data, columns=['lon', 'lat', 'strength']) self.domain = (0,0, 100, 100) def test_suppress_non_optimal(self): with warnings.catch_warnings(record=True) as w: def test_suppress(self): mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=25) self.assertEqual(len(w), 1) self.assertEqual(w[0].category, UserWarning) self.assertEqual(mask.sum(), 25) self.assertEqual(mask.sum(), 28) def test_suppress(self): mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=30) self.assertIn(mask.sum(), list(range(27, 35))) mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=30, error_k=.15) self.assertEqual(mask.sum(), 34) def test_suppress_non_optimal(self): with warnings.catch_warnings(record=True) as w: mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=101) mask, k = cpu_outlier_detector.spatial_suppression(self.df, self.domain, k=30) self.assertEqual(len(w), 1) self.assertEqual(k, 59) self.assertTrue(issubclass(w[0].category, UserWarning)) class testSuppressionRanges(unittest.TestCase): Loading @@ -79,21 +76,21 @@ class testSuppressionRanges(unittest.TestCase): cls.r = np.random.RandomState(12345) def test_min_max(self): df = pd.DataFrame(self.r.uniform(0,2,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (1.5,1.5), k = 1) df = pd.DataFrame(self.r.uniform(0,2,(500, 3)), columns=['lon', 'lat', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 1.5, 1.5), k = 1) self.assertEqual(len(df[mask]), 1) def test_point_overload(self): df = pd.DataFrame(self.r.uniform(0,15,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (15,15), k = 200) self.assertEqual(len(df[mask]), 69) df = pd.DataFrame(self.r.uniform(0,15,(500, 3)), columns=['lon', 'lat', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 15, 15), k = 200) self.assertEqual(len(df[mask]), 195) def test_small_distribution(self): df = pd.DataFrame(self.r.uniform(0,25,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (25,25), k = 25) self.assertEqual(len(df[mask]), 28) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 25, 25), k = 25, xkey='x', ykey='y') self.assertEqual(len(df[mask]), 25) def test_normal_distribution(self): df = pd.DataFrame(self.r.uniform(0,100,(500, 3)), columns=['x', 'y', 'strength']) mask, k = cpu_outlier_detector.spatial_suppression(df, (100,100), k = 15) mask, k = cpu_outlier_detector.spatial_suppression(df, (0, 0, 100, 100), k = 15, xkey='x', ykey='y') self.assertEqual(len(df[mask]), 17)
conda/meta.yaml +1 −1 Original line number Diff line number Diff line package: name: autocnet version: 0.2.0 version: 0.2.1 channels: - conda-forge Loading
setup.py +1 −4 Original line number Diff line number Diff line Loading @@ -6,9 +6,6 @@ from autocnet.examples import available with open('README.rst', 'r') as f: long_description = f.read() VERSION = autocnet.__version__ def setup_package(): examples = set() for i in available(): Loading @@ -23,7 +20,7 @@ def setup_package(): setup( name = "autocnet", version = VERSION, version = '0.2.1', author = "Jay Laura", author_email = "jlaura@usgs.gov", description = ("I/O API to support planetary data formats."), Loading