Adds feature save capabilities to npy (88e8ce63) · Commits · aflab / astrogeology / Autocnet

autocnet/graph/network.py

+12 −28

Changes for autocnet/graph/network.py: 12 added lines, 28 removed lines.
Original line number	Diff line number	Diff line
		@@ -7,12 +7,13 @@ import dill as pickle
		import networkx as nx
		import pandas as pd

		from plio.io import io_hdf, io_json, io_autocnetgraph
		from plio.io import io_hdf, io_json
		from plio.utils import utils as io_utils
		from plio.io.io_gdal import GeoDataset
		from autocnet.graph import markov_cluster
		from autocnet.graph.edge import Edge
		from autocnet.graph.node import Node
		from autocnet.io import network as io_network
		from autocnet.vis.graph_view import plot_graph, cluster_plot


		@@ -235,7 +236,7 @@ class CandidateGraph(nx.Graph):
		image = node.get_array()
		node.extract_features(image, args, *kwargs),

		def save_features(self, out_path, nodes=[]):
		def save_features(self, out_path, nodes=[], **kwargs):
		"""

		Save the features (keypoints and descriptors) for the
		@@ -251,24 +252,12 @@ class CandidateGraph(nx.Graph):
		of nodes to save features for. If empty, save for all nodes
		"""

		if os.path.exists(out_path):
		mode = 'a'
		else:
		mode = 'w'

		hdf = io_hdf.HDFDataset(out_path, mode=mode)

		# Cleaner way to do this?
		if nodes:
		for i, n in self.subgraph(nodes).nodes_iter(data=True):
		n.save_features(hdf)
		else:
		for i, n in self.nodes_iter(data=True):
		n.save_features(hdf)

		hdf = None
		if nodes and not i in nodes:
		continue
		n.save_features(out_path, **kwargs)

		def load_features(self, in_path, nodes=[], nfeatures=None):
		def load_features(self, in_path, nodes=[], nfeatures=None, **kwargs):
		"""
		Load features (keypoints and descriptors) for the
		specified nodes.
		@@ -282,16 +271,11 @@ class CandidateGraph(nx.Graph):
		of nodes to load features for. If empty, load features
		for all nodes
		"""
		hdf = io_hdf.HDFDataset(in_path, 'r')

		if nodes:
		for i, n in self.subgraph(nodes).nodes_iter(data=True):
		n.load_features(hdf)
		else:
		for i, n in self.nodes_iter(data=True):
		n.load_features(hdf)

		hdf = None
		if nodes and not i in nodes:
		continue
		else:
		n.load_features(in_path, **kwargs)

		def match(self, args, *kwargs):
		"""
		@@ -555,7 +539,7 @@ class CandidateGraph(nx.Graph):
		filename : str
		The relative or absolute PATH where the network is saved
		"""
		io_autocnetgraph.save(self, filename)
		io_network.save(self, filename)

		def plot(self, ax=None, **kwargs): # pragma: no cover
		"""

autocnet/graph/node.py

+14 −8

Changes for autocnet/graph/node.py: 14 added lines, 8 removed lines.
Original line number	Diff line number	Diff line
		@@ -306,25 +306,29 @@ class Node(dict, MutableMapping):
		in_path : str or object
		PATH to the hdf file or a HDFDataset object handle

		format : {'npy', 'hdf5'}
		format : {'npy', 'hdf'}
		"""
		if format == 'npy':
		io_keypoints.from_npy(in_path, self)
		elif format == 'hdf5':
		io_keypoints.from_hdf(in_path, self)
		keypoints, descriptors = io_keypoints.from_npy(in_path)
		elif format == 'hdf':
		keypoints, descriptors = io_keypoints.from_hdf(in_path,
		key=self['image_name'])

		self._keypoints = keypoints
		self.descriptors = descriptors

		def save_features(self, out_path, format='npy'):
		"""
		Save the extracted keypoints and descriptors to
		the given HDF5 file.
		the given HDF5 file. By default, the .npz files are saved
		along side the image, e.g. in the same folder as the image.

		Parameters
		----------
		out_path : str or object
		PATH to the hdf file or a HDFDataset object handle

		format : {'npy', 'hdf5'}
		format : {'npy', 'hdf'}
		The desired output format.
		"""

		@@ -333,9 +337,11 @@ class Node(dict, MutableMapping):
		return

		if format == 'hdf':
		io_keypoints.to_hdf(out_path, self)
		io_keypoints.to_hdf(self._keypoints, self.descriptors, out_path,
		key=self['image_name'])
		elif format == 'npy':
		io_keypoints.to_npy(out_path, self)
		io_keypoints.to_npy(self._keypoints, self.descriptors,
		out_path)
		else:
		warnings.warn('Unknown keypoint output format.')

autocnet/graph/tests/test_network.py

+4 −3

Changes for autocnet/graph/tests/test_network.py: 4 added lines, 3 removed lines.
Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@ from .. import network

		sys.path.insert(0, os.path.abspath('..'))


		@pytest.fixture()
		def graph():
		basepath = get_path('Apollo15')
		@@ -72,11 +73,11 @@ def test_save_load_features(tmpdir, graph):
		allout = tmpdir.join("all_out.hdf")
		oneout = tmpdir.join("one_out.hdf")

		graph.save_features(allout.strpath)
		graph.save_features(oneout.strpath, nodes=[1])
		graph.save_features(allout.strpath, format='hdf')
		graph.save_features(oneout.strpath, nodes=[1], format='hdf')

		graph_no_features = graph.copy()
		graph_no_features.load_features(allout.strpath, nodes=[1])
		graph_no_features.load_features(allout.strpath, nodes=[1], format='hdf')
		assert graph.node[1].get_keypoints().all().all() == graph_no_features.node[1].get_keypoints().all().all()

		def test_filter(graph):

autocnet/io/keypoints.py

+107 −21

Changes for autocnet/io/keypoints.py: 107 added lines, 21 removed lines.
Original line number	Diff line number	Diff line
		import os

		import numpy as np
		import pandas as pd
		from plio.io import io_hdf

		def from_hdf(in_path, node):
		from autocnet.utils import utils

		def from_hdf(in_path, key=None):
		"""
		For a given node, load the keypoints and descriptors from a hdf5 file.

		Parameters
		----------
		in_path : str
		handle to the file

		key : str
		An optional path into the HDF5. For example key='image_name', will
		search /image_name/descriptors for the descriptors.

		Returns
		-------
		keypoints : DataFrame
		A pandas dataframe of keypoints.

		descriptors : ndarray
		A numpy array of descriptors
		"""
		if isinstance(in_path, str):
		hdf = io_hdf.HDFDataset(in_path, mode='r')
		else:
		hdf = in_path

		node.descriptors = hdf['{}/descriptors'.format(node['image_name'])][:]
		raw_kps = hdf['{}/keypoints'.format(node['image_name'])][:]
		if key:
		outd = '{}/descriptors'.format(key)
		outk = '{}/keypoints'.format(key)
		else:
		outd = '/descriptors'
		outk = '/keypoints'

		descriptors = hdf[outd][:]
		raw_kps = hdf[outk][:]
		index = raw_kps['index']
		clean_kps = utils.remove_field_name(raw_kps, 'index')
		columns = clean_kps.dtype.names

		allkps = pd.DataFrame(data=clean_kps, columns=columns, index=index)

		if 'response' in allkps.columns:
		node._keypoints = allkps.sort_values(by='response', ascending=False)
		elif 'size' in allkps.columns:
		node._keypoints = allkps.sort_values(by='size', ascending=False)
		if isinstance(in_path, str):
		hdf = None

		def to_hdf(out_path, node):
		return allkps, descriptors


		def to_hdf(keypoints, descriptors, out_path, key=None):
		"""
		Save keypoints and descriptors to HDF at a given out_path at either
		the root or at some arbitrary path given by a key.

		Parameters
		----------
		keypoints : DataFrame
		Pandas dataframe of keypoints

		descriptors : ndarray
		of feature descriptors

		out_path : str
		to the HDF5 file

		key : str
		path within the HDF5 file. If given, the keypoints and descriptors
		are save at <key>/keypoints and <key>/descriptors respectively.
		"""
		# If the out_path is a string, access the HDF5 file
		if isinstance(out_path, str):
		if os.path.exists(out_path):
		@@ -36,12 +84,18 @@ def to_hdf(out_path, node):
		hdf = out_path

		#try:
		hdf.create_dataset('{}/descriptors'.format(node['image_name']),
		data=node.descriptors,
		if key:
		outd = '{}/descriptors'.format(key)
		outk = '{}/keypoints'.format(key)
		else:
		outd = '/descriptors'
		outk = '/keypoints'
		hdf.create_dataset(outd,
		data=descriptors,
		compression=io_hdf.DEFAULT_COMPRESSION,
		compression_opts=io_hdf.DEFAULT_COMPRESSION_VALUE)
		hdf.create_dataset('{}/keypoints'.format(node['image_name']),
		data=hdf.df_to_sarray(node._keypoints.reset_index()),
		hdf.create_dataset(outk,
		data=hdf.df_to_sarray(keypoints.reset_index()),
		compression=io_hdf.DEFAULT_COMPRESSION,
		compression_opts=io_hdf.DEFAULT_COMPRESSION_VALUE)
		#except:
		@@ -53,13 +107,45 @@ def to_hdf(out_path, node):
		if isinstance(out_path, str):
		hdf = None

		def from_npy(in_path, node):
		def from_npy(in_path):
		"""
		Load keypoints and descriptors from a .npz file.

		Parameters
		----------
		in_path : str
		PATH to the npz file

		Returns
		-------
		keypoints : DataFrame
		of keypoints

		descriptors : ndarray
		of feature descriptors
		"""
		nzf = np.load(in_path)
		node.descriptors = nzf['descriptors']
		node._keypoints = pd.DataFrame(nzf['_keypoints'], index=nzf['_keypoints_idx'], columns=nzf['_keypoints_columns'])

		def to_npy(out_path, node):
		np.savez(out_path, descriptors=node.descriptors,
		_keypoints=data._keypoints,
		_keypoints_idx=data._keypoints.index,
		_keypoints_columns=data._keypoints.columns)
		descriptors = nzf['descriptors']
		keypoints = pd.DataFrame(nzf['keypoints'], index=nzf['keypoints_idx'], columns=nzf['keypoints_columns'])

		return keypoints, descriptors

		def to_npy(keypoints, descriptors, out_path):
		"""
		Save keypoints and descriptors to a .npz file at some out_path

		Parameters
		----------
		keypoints : DataFrame
		of keypoints

		descriptors : ndarray
		of feature descriptors

		out_path : str
		PATH and filename to save the features
		"""
		np.savez(out_path, descriptors=descriptors,
		keypoints=keypoints,
		keypoints_idx=keypoints.index,
		keypoints_columns=keypoints.columns)

autocnet/io/network.py

0 → 100644

+123 −0

Changes for autocnet/io/network.py: 123 added lines, 0 removed lines.
Original line number	Diff line number	Diff line
		from io import BytesIO
		import json
		import os
		import warnings
		from zipfile import ZipFile

		from networkx.readwrite import json_graph
		import numpy as np
		import pandas as pd

		import autocnet


		class NumpyEncoder(json.JSONEncoder):
		def default(self, obj):
		"""If input object is an ndarray it will be converted into a dict
		holding dtype, shape and the data, base64 encoded.
		"""
		if isinstance(obj, np.ndarray):
		return dict(__ndarray__= obj.tolist(),
		dtype=str(obj.dtype),
		shape=obj.shape)
		# Let the base class default method raise the TypeError
		return json.JSONEncoder.default(self, obj)

		def save(network, projectname):
		"""
		Save an AutoCNet candiate graph to disk in a compressed file. The
		graph adjacency structure is stored as human readable JSON and all
		potentially large numpy arrays are stored as compressed binary. The
		project archive is a standard .zip file that can have any ending,
		e.g., <projectname>.project, <projectname>.zip, <projectname>.myname.

		TODO: This func. writes a intermediary .npz to disk when saving. Can
		we write the .npz to memory?

		Parameters
		----------
		network : object
		The AutoCNet Candidate Graph object

		projectname : str
		The PATH to the output file.
		"""
		# Convert the graph into json format
		js = json_graph.node_link_data(network)

		with ZipFile(projectname, 'w') as pzip:
		js_str = json.dumps(js, cls=NumpyEncoder, sort_keys=True, indent=4)
		pzip.writestr('graph.json', js_str)

		# Write the array node_attributes to hdf
		for n, data in network.nodes_iter(data=True):
		grp = data['node_id']
		np.savez('{}.npz'.format(data['node_id']),
		descriptors=data.descriptors,
		_keypoints=data._keypoints,
		_keypoints_idx=data._keypoints.index,
		_keypoints_columns=data._keypoints.columns)
		pzip.write('{}.npz'.format(data['node_id']))
		os.remove('{}.npz'.format(data['node_id']))

		# Write the array edge attributes to hdf
		for s, d, data in network.edges_iter(data=True):
		if s > d:
		s, d = d, s
		grp = str((s,d))
		np.savez('{}_{}.npz'.format(s, d),
		matches=data.matches,
		matches_idx=data.matches.index,
		matches_columns=data.matches.columns,
		_masks=data._masks,
		_masks_idx=data._masks.index,
		_masks_columns=data._masks.columns)
		pzip.write('{}_{}.npz'.format(s, d))
		os.remove('{}_{}.npz'.format(s, d))

		def json_numpy_obj_hook(dct):
		"""Decodes a previously encoded numpy ndarray with proper shape and dtype.

		:param dct: (dict) json encoded ndarray
		:return: (ndarray) if input was an encoded ndarray
		"""
		if isinstance(dct, dict) and '__ndarray__' in dct:
		data = np.asarray(dct['__ndarray__'])
		return np.frombuffer(data, dct['dtype']).reshape(dct['shape'])
		return dct

		def load(projectname):

		with ZipFile(projectname, 'r') as pzip:
		# Read the graph object
		with pzip.open('graph.json', 'r') as g:
		data = json.loads(g.read().decode(),object_hook=json_numpy_obj_hook)

		cg = autocnet.graph.network.CandidateGraph()
		Edge = autocnet.graph.edge.Edge
		Node = autocnet.graph.node.Node
		# Reload the graph attributes
		cg.graph = data['graph']
		# Handle nodes
		for d in data['nodes']:
		n = Node(image_name=d['image_name'], image_path=d['image_path'], node_id=d['id'])
		n['hash'] = d['hash']
		# Load the byte stream for the nested npz file into memory and then unpack
		n.load_features(BytesIO(pzip.read('{}.npz'.format(d['id']))))
		cg.add_node(d['node_id'])
		cg.node[d['node_id']] = n
		for e in data['links']:
		cg.add_edge(e['source'], e['target'])
		edge = Edge()
		edge.source = cg.node[e['source']]
		edge.destination = cg.node[e['target']]
		edge['fundamental_matrix'] = e['fundamental_matrix']
		edge['weight'] = e['weight']
		nzf = np.load(BytesIO(pzip.read('{}_{}.npz'.format(e['source'], e['target']))))

		edge._masks = pd.DataFrame(nzf['_masks'], index=nzf['_masks_idx'], columns=nzf['_masks_columns'])
		edge.matches = pd.DataFrame(nzf['matches'], index=nzf['matches_idx'], columns=nzf['matches_columns'])
		# Add a mock edge
		cg.edge[e['source']][e['target']] = edge

		return cg