move binary file output to virtual file passed via MPI to process 0, thread 0... (2244fd0c) · Commits · Giacomo Mulas / NP_TMcode

src/cluster/cluster.cpp

+210 −131

File changed.

Preview size limit exceeded, changes collapsed.

src/include/file_io.h

+109 −98

Original line number	Diff line number	Diff line
		@@ -252,109 +252,120 @@ public:
		*/
		void mpisend(const mixMPI *mpidata);
		};
		#endif


		// /*! \class VirtualBinaryLine
		// *
		// * \brief Virtual representation of a binary file line.
		// */
		// class VirtualBinaryLine {
		/*! \class VirtualBinaryLine
		*
		* \brief Virtual representation of a binary file line.
		*/
		class VirtualBinaryLine {
		// protected:
		// //! \brief The pointer to the piece of data to be written, cast to char *
		// char *_data_pointer;
		// //! \brief the size of the data block.
		// size_t _data_size;
		// /*! \brief VirtualBinaryLine instance constructor.
		// *
		// * \param mydata: `int, double, long, float, complex, or dcomplex`piece of data to put in the line.
		// */
		// VirtualBinaryLine(int mydata);
		// VirtualBinaryLine(long mydata);
		// VirtualBinaryLine(float mydata);
		// VirtualBinaryLine(double mydata);

		public:
		//! \brief The pointer to the piece of data to be written, cast to char *
		char *_data_pointer;
		//! \brief the size of the data block.
		size_t _data_size;
		//! \brief Read only view of `_data_pointer`.
		const char* data_pointer = _data_pointer;
		//! \brief Read only view of `_data_size`.
		const size_t & data_size = _data_size;

		/*! \brief VirtualBinaryLine instance constructor.
		*
		* \param mydata: `int, double, long, float, complex, or dcomplex`piece of data to put in the line.
		*/
		VirtualBinaryLine(int mydata);
		VirtualBinaryLine(long mydata);
		VirtualBinaryLine(float mydata);
		VirtualBinaryLine(double mydata);
		// VirtualBinaryLine(complex mydata);
		// VirtualBinaryLine(dcomplex mydata);
		VirtualBinaryLine(dcomplex mydata);

		// /*! \brief VirtualBinaryLine copy constructor.
		// *
		// * \param rhs: `const VirtualBinaryLine&` Reference to a VirtualBinaryLine instance.
		// */
		// VirtualBinaryLine(const VirtualBinaryLine& rhs);
		/*! \brief VirtualBinaryLine copy constructor.
		*
		* \param rhs: `const VirtualBinaryLine&` Reference to a VirtualBinaryLine instance.
		*/
		VirtualBinaryLine(const VirtualBinaryLine& rhs);

		// /*! \brief VirtualBinaryLine instance constructor copying all contents off MPISend() calls from MPI process rr.
		// *
		// * \param mpidata: `mixMPI *` pointer to MPI data structure.
		// * \param rr: `int` rank of the MPI process sending the data.
		// */
		// VirtualBinaryLine(const mixMPI *mpidata, int rr);
		/*! \brief VirtualBinaryLine instance constructor copying all contents off MPISend() calls from MPI process rr.
		*
		* \param mpidata: `mixMPI *` pointer to MPI data structure.
		* \param rr: `int` rank of the MPI process sending the data.
		*/
		VirtualBinaryLine(const mixMPI *mpidata, int rr);

		// /*! \brief VirtualBinaryLine instance destroyer.
		// */
		// ~VirtualBinaryLine();
		/*! \brief VirtualBinaryLine instance destroyer.
		*/
		~VirtualBinaryLine();

		// /*! \brief Send VirtualBinaryLine instance to MPI process 0 via MPISend() calls.
		// *
		// * \param mpidata: `mixMPI *` pointer to MPI data structure.
		// */
		// void mpisend(const mixMPI *mpidata);
		// };
		/*! \brief Send VirtualBinaryLine instance to MPI process 0 via MPISend() calls.
		*
		* \param mpidata: `mixMPI *` pointer to MPI data structure.
		*/
		void mpisend(const mixMPI *mpidata);
		};


		// /*! \class VirtualBinaryFile
		// *
		// * \brief Virtual representation of a binary file.
		// */
		// class VirtualBinaryFile {
		// protected:
		// //! \brief The number of lines.
		// // int32_t _num_lines;
		/*! \class VirtualBinaryFile
		*
		* \brief Virtual representation of a binary file.
		*/
		class VirtualBinaryFile {
		protected:
		//! \brief The number of lines.
		// int32_t _num_lines;
		// //! \brief A vector of strings representing the file lines.
		// std::vector<VirtualBinaryLine> *_file_lines;

		// public:
		// // const int32_t &num_lines = _num_lines;
		// /*! \brief VirtualBinaryFile instance constructor.
		// *
		// * \param lines: `int32_t` Number of lines, if known in advance (optional, default is 0).
		// */
		// VirtualBinaryFile(int32_t lines = 0);
		public:
		//! \brief A vector of strings representing the file lines.
		std::vector<VirtualBinaryLine> *_file_lines;
		// const int32_t &num_lines = _num_lines;
		/*! \brief VirtualBinaryFile empty instance constructor.
		*
		*/
		VirtualBinaryFile();

		// /*! \brief VirtualBinaryFile copy constructor.
		// *
		// * \param rhs: `const VirtualBinaryFile&` Reference to a VirtualBinaryFile instance.
		// */
		// VirtualBinaryFile(const VirtualBinaryFile& rhs);
		/*! \brief VirtualBinaryFile copy constructor.
		*
		* \param rhs: `const VirtualBinaryFile&` Reference to a VirtualBinaryFile instance.
		*/
		VirtualBinaryFile(const VirtualBinaryFile& rhs);

		// /*! \brief VirtualBinaryFile instance constructor copying all contents off MPISend() calls from MPI process rr.
		// *
		// * \param mpidata: `mixMPI *` pointer to MPI data structure.
		// * \param rr: `int` rank of the MPI process sending the data.
		// */
		// VirtualBinaryFile(const mixMPI *mpidata, int rr);
		/*! \brief VirtualBinaryFile instance constructor copying all contents off MPISend() calls from MPI process rr.
		*
		* \param mpidata: `mixMPI *` pointer to MPI data structure.
		* \param rr: `int` rank of the MPI process sending the data.
		*/
		VirtualBinaryFile(const mixMPI *mpidata, int rr);

		// /*! \brief VirtualBinaryFile instance destroyer.
		// */
		// ~VirtualBinaryFile();
		/*! \brief VirtualBinaryFile instance destroyer.
		*/
		~VirtualBinaryFile();

		// /*! \brief Append another VirtualBinaryFile at the end of the current instance.
		// *
		// * \param rhs: `const VirtualBinaryFile&` Reference to the VirtualBinaryFile to be appended.
		// */
		// void append(VirtualBinaryFile& rhs);
		/*! \brief Append another VirtualBinaryFile at the end of the current instance.
		*
		* \param rhs: `const VirtualBinaryFile&` Reference to the VirtualBinaryFile to be appended.
		*/
		void append(VirtualBinaryFile& rhs);

		// /*! \brief Append a line at the end of the file.
		// *
		// * \param line: `const string&` Reference to a string representing the line.
		// */
		// void append_line(const std::string& line);
		/*! \brief Append a line at the end of the file.
		*
		* \param line: `const string&` Reference to a string representing the line.
		*/
		void append_line(const VirtualBinaryLine& line);

		// /*! \brief Append the contents of the VirtualBinaryFile to a physical file on disk.
		// *
		// * \param file_name: `const string&` Name of the file to append contents to.
		// * \return result: `int` A result code (0 if successful).
		// */
		// int append_to_disk(const std::string& file_name);
		/*! \brief Append the contents of the VirtualBinaryFile to a physical file on disk.
		*
		* \param file_name: `const string&` Name of the file to append contents to.
		* \return result: `int` A result code (0 if successful).
		*/
		int append_to_disk(const std::string& file_name);

		// /*! \brief Insert another VirtualBinaryFile at a given position.
		// *
		@@ -373,25 +384,25 @@ public:
		// */
		// int insert(int32_t position, VirtualBinaryFile& rhs, int32_t start = 0, int32_t end = 0);

		// /*! \brief Get the number of lines in the current instance.
		// *
		// * \return size: `int32_t` The number of lines in the VirtualBinaryFile instance.
		// */
		// int32_t number_of_lines() { return _file_lines->size(); }
		/*! \brief Get the number of lines in the current instance.
		*
		* \return size: `int32_t` The number of lines in the VirtualBinaryFile instance.
		*/
		int32_t number_of_lines() { return _file_lines->size(); }

		// /*! \brief Write virtual file contents to a real file on disk.
		// *
		// * \param file_name: `const string&` Name of the file to append contents to.
		// * \return result: `int` A result code (0 if successful).
		// */
		// int write_to_disk(const std::string& file_name);
		/*! \brief Write virtual file contents to a real file on disk.
		*
		* \param file_name: `const string&` Name of the file to append contents to.
		* \return result: `int` A result code (0 if successful).
		*/
		int write_to_disk(const std::string& file_name);

		// /*! \brief Send VirtualBinaryFile instance to MPI process 0 via MPISend() calls.
		// *
		// * \param mpidata: `mixMPI *` pointer to MPI data structure.
		// */
		// void mpisend(const mixMPI *mpidata);
		// };
		//#endif
		/*! \brief Send VirtualBinaryFile instance to MPI process 0 via MPISend() calls.
		*
		* \param mpidata: `mixMPI *` pointer to MPI data structure.
		*/
		void mpisend(const mixMPI *mpidata);
		};
		#endif

src/libnptm/file_io.cpp

+195 −31

Original line number	Diff line number	Diff line
		@@ -16,6 +16,10 @@
		#include "../include/List.h"
		#endif

		#ifndef INCLUDE_TYPES_H_
		#include "../include/types.h"
		#endif

		#ifndef INCLUDE_FILE_IO_H_
		#include "../include/file_io.h"
		#endif
		@@ -301,6 +305,7 @@ int VirtualAsciiFile::append_to_disk(const std::string& file_name) {
		for (vector<string>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {
		output_file << *it;
		}
		output_file.close();
		} else {
		result = 1;
		}
		@@ -334,6 +339,7 @@ int VirtualAsciiFile::write_to_disk(const std::string& file_name) {
		for (vector<string>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {
		output_file << *it;
		}
		output_file.close();
		} else {
		result = 1;
		}
		@@ -361,45 +367,203 @@ void VirtualAsciiFile::mpisend(const mixMPI *mpidata) {
		/* >>> End of VirtualAsciiFile class implementation <<< */

		// /* >>> VirtualBinaryLine class implementation <<< */
		// VirtualBinaryLine::VirtualBinaryLine(int mydata) {
		// _data_size = sizeof(mydata)
		// int *buffer = malloc(_data_size);
		// *buffer = mydata;
		// _data_pointer = reinterpret_cast<char *>(buffer);
		// }
		VirtualBinaryLine::VirtualBinaryLine(int mydata) {
		_data_size = sizeof(mydata);
		int buffer = (int ) malloc(_data_size);
		*buffer = mydata;
		_data_pointer = reinterpret_cast<char *>(buffer);
		}

		// VirtualBinaryLine::VirtualBinaryLine(double mydata) {
		// _data_size = sizeof(mydata)
		// double *buffer = malloc(_data_size);
		// *buffer = mydata;
		// _data_pointer = reinterpret_cast<char *>(buffer);
		// }
		VirtualBinaryLine::VirtualBinaryLine(double mydata) {
		_data_size = sizeof(mydata);
		double buffer = (double ) malloc(_data_size);
		*buffer = mydata;
		_data_pointer = reinterpret_cast<char *>(buffer);
		}

		// VirtualBinaryLine::VirtualBinaryLine(float mydata) {
		// _data_size = sizeof(mydata)
		// float *buffer = malloc(_data_size);
		// *buffer = mydata;
		// _data_pointer = reinterpret_cast<char *>(buffer);
		// }
		VirtualBinaryLine::VirtualBinaryLine(float mydata) {
		_data_size = sizeof(mydata);
		float buffer = (float ) malloc(_data_size);
		*buffer = mydata;
		_data_pointer = reinterpret_cast<char *>(buffer);
		}

		// VirtualBinaryLine::VirtualBinaryLine(long mydata) {
		// _data_size = sizeof(mydata)
		// long *buffer = malloc(_data_size);
		VirtualBinaryLine::VirtualBinaryLine(long mydata) {
		_data_size = sizeof(mydata);
		long buffer = (long ) malloc(_data_size);
		*buffer = mydata;
		_data_pointer = reinterpret_cast<char *>(buffer);
		}

		VirtualBinaryLine::VirtualBinaryLine(dcomplex mydata) {
		_data_size = sizeof(mydata);
		dcomplex buffer = (dcomplex ) malloc(_data_size);
		*buffer = mydata;
		_data_pointer = reinterpret_cast<char *>(buffer);
		}

		// VirtualBinaryLine::VirtualBinaryLine(complex mydata) {
		// _data_size = sizeof(mydata);
		// complex buffer = (complex ) malloc(_data_size);
		// *buffer = mydata;
		// _data_pointer = reinterpret_cast<char *>(buffer);
		// }

		// VirtualBinaryLine::VirtualBinaryLine(dcomplex mydata) {
		// _data_size = sizeof(mydata)
		// dcomplex *buffer = malloc(_data_size);
		// *buffer = mydata;
		// _data_pointer = reinterpret_cast<char *>(buffer);
		VirtualBinaryLine::VirtualBinaryLine(const VirtualBinaryLine& rhs) {
		_data_size = rhs._data_size;
		_data_pointer = reinterpret_cast<char *>(malloc(rhs._data_size));
		memcpy(_data_pointer, rhs._data_pointer, _data_size);
		}

		#ifdef MPI_VERSION
		VirtualBinaryLine::VirtualBinaryLine(const mixMPI *mpidata, int rr) {
		// receive mysize from MPI process rr
		int32_t mysize;
		MPI_Recv(&mysize, 1, MPI_INT32_T, rr, 10, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
		_data_size = mysize;
		// allocate the buffer accordingly
		_data_pointer = reinterpret_cast<char *>(malloc(mysize));
		// receive the char buffer
		MPI_Recv(_data_pointer, mysize, MPI_CHAR, rr, 10, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
		}

		#endif

		VirtualBinaryLine::~VirtualBinaryLine() {
		if (_data_pointer != NULL) {
		free(_data_pointer);
		_data_pointer = NULL;
		}
		}

		#ifdef MPI_VERSION
		void VirtualBinaryLine::mpisend(const mixMPI *mpidata) {
		// Send VirtualBinaryLine instance to MPI process 0 via MPISend() calls
		// first send the size
		int32_t mysize = _data_size;
		MPI_Send(&mysize, 1, MPI_INT32_T, 0, 10, MPI_COMM_WORLD);
		// now send the data
		MPI_Send(_data_pointer, mysize, MPI_CHAR, 0, 10, MPI_COMM_WORLD);
		}
		#endif
		/* >>> End of VirtualBinaryLine class implementation <<< */


		/* >>> VirtualBinaryFile class implementation <<< */
		VirtualBinaryFile::VirtualBinaryFile() {
		_file_lines = new vector<VirtualBinaryLine>();
		}

		VirtualBinaryFile::VirtualBinaryFile(const VirtualBinaryFile& rhs) {
		// _num_lines = rhs._num_lines;
		_file_lines = new vector<VirtualBinaryLine>();
		for (vector<VirtualBinaryLine>::iterator it = rhs._file_lines->begin(); it != rhs._file_lines->end(); ++it) {
		_file_lines->push_back(VirtualBinaryLine(*it));
		}
		}

		#ifdef MPI_VERSION
		VirtualBinaryFile::VirtualBinaryFile(const mixMPI *mpidata, int rr) {
		// receive _num_lines from MPI process rr
		int32_t num_lines;
		MPI_Recv(&num_lines, 1, MPI_INT32_T, rr, 10, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
		_file_lines = new vector<VirtualBinaryLine>();
		// loop over data to receive
		for (int32_t zi=0; zi<num_lines; zi++) {
		// receive the line of data
		_file_lines->push_back(VirtualBinaryLine(mpidata, rr));
		}
		}
		#endif

		VirtualBinaryFile::~VirtualBinaryFile() {
		// is it necessary to pop them out one by one? isn't there the dedicated method of std::vector to clean the vector?
		// besides, shouldn't this be done anyway by the destructor of std:vector?
		// for (vector<VirtualBinaryLine>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {
		// delete it;
		// }
		while (!_file_lines->size() > 0) {
		_file_lines->pop_back();
		}
		if (_file_lines != NULL) delete _file_lines;
		}

		// VirtualBinaryLine::VirtualBinaryLine(complex mydata) {
		// _data_size = sizeof(mydata)
		// complex *buffer = malloc(_data_size);
		// *buffer = mydata;
		// _data_pointer = reinterpret_cast<char *>(buffer);
		void VirtualBinaryFile::append(VirtualBinaryFile& rhs) {
		// concatenate the virtualasciifile pointed by rhs to the current one
		// can't we use the dedicated method insert of std::vector to do the appending, instead of an explicit loop?
		for (vector<VirtualBinaryLine>::iterator it = rhs._file_lines->begin(); it != rhs._file_lines->end(); ++it) {
		_file_lines->push_back(VirtualBinaryLine(*it));
		}
		}

		void VirtualBinaryFile::append_line(const VirtualBinaryLine& line) {
		// would it be worth reimplementing a sprintf-like method, so that we can give it all the arguments we would give to sprintf and get rid of the intermediate buffer completely?
		// append a line of output to the virtualasciifile
		_file_lines->push_back(VirtualBinaryLine(line));
		}

		int VirtualBinaryFile::append_to_disk(const std::string& file_name) {
		// dump to disk the contents of the virtualasciifile, appending at the end of the given file_name
		int result = 0;
		fstream output_file;
		output_file.open(file_name, ios::app \| ios::binary);
		if (output_file.is_open()) {
		for (vector<VirtualBinaryLine>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {
		output_file.write(it->_data_pointer, it->_data_size);
		}
		output_file.close();
		} else {
		result = 1;
		}
		return result;
		}

		// int VirtualBinaryFile::insert(int32_t position, VirtualBinaryFile& rhs, int32_t start, int32_t end) {
		// int result = 0;
		// if (start == 0 && end == 0) {
		// end = rhs.number_of_lines();
		// }
		// int32_t final_index = position + end - start;
		// if (final_index <= number_of_lines()) {
		// for (int32_t li = start; li < end; li++) {
		// // since here we are replacing the previous placeholder empty strings, make sure they are properly released when they are replaced (i.e. try it with a simple hello world example and pass it through valgrind)
		// VirtualBinaryLine templine = VirtualBinaryLine(rhs._file_lines->at(li));
		// _file_lines->at(position++) = templine;
		// }
		// } else {
		// // ERROR: target file is too long;
		// result = 1;
		// }
		// return result;
		// }

		int VirtualBinaryFile::write_to_disk(const std::string& file_name) {
		// dump to disk the contents of the virtualasciifile, replacing the given file_name
		int result = 0;
		fstream output_file;
		output_file.open(file_name, ios::out \| ios::binary);
		if (output_file.is_open()) {
		for (vector<VirtualBinaryLine>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {
		output_file.write(it->_data_pointer, it->_data_size);
		}
		output_file.close();
		} else {
		result = 1;
		}
		return result;
		}

		#ifdef MPI_VERSION
		void VirtualBinaryFile::mpisend(const mixMPI *mpidata) {
		// Send VirtualBinaryFile instance to MPI process 0 via MPISend() calls
		// first send the size
		int32_t num_lines = _file_lines->size();
		MPI_Send(&num_lines, 1, MPI_INT32_T, 0, 10, MPI_COMM_WORLD);
		// now loop over data to send
		for (vector<VirtualBinaryLine>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {
		it->mpisend(mpidata);
		}
		}
		#endif

		/* >>> End of VirtualBinaryFile class implementation <<< */