Commit 9e49e770 authored by Mulas, Giacomo's avatar Mulas, Giacomo
Browse files

Virtual files are dumped to disk at each iteration in MPI processes,... 

Virtual files are dumped to disk at each iteration in MPI processes, drastically reducing memory usage
parent 1c2d1352

src/cluster/cluster.cpp

+1017 −1103

File changed.

Preview size limit exceeded, changes collapsed.

src/libnptm/Commons.cpp

+2 −0
Original line number Diff line number Diff line
@@ -1184,6 +1184,8 @@ ClusterIterationData::ClusterIterationData(const ClusterIterationData& rhs) { 
  vk = rhs.vk;

  firstxi = rhs.firstxi;

  lastxi = rhs.lastxi;

  xiblock = rhs.xiblock;

  number_of_scales = rhs.number_of_scales;

}



#ifdef MPI_VERSION

src/libnptm/file_io.cpp

+61 −47
Original line number Diff line number Diff line
@@ -277,9 +277,10 @@ VirtualAsciiFile::VirtualAsciiFile(const mixMPI *mpidata, int rr) { 
VirtualAsciiFile::~VirtualAsciiFile() {

  // is it necessary to pop them out one by one? isn't there the dedicated method of std::vector to clean the vector?

  // besides, shouldn't this be done anyway by the destructor of std:vector?

  while (!_file_lines->size() > 0) {

    _file_lines->pop_back();

  }

  _file_lines->clear();

  // while (_file_lines->size() > 0) {

  //   _file_lines->pop_back();

  // }

  if (_file_lines != NULL) delete _file_lines;

}
@@ -300,6 +301,7 @@ void VirtualAsciiFile::append_line(const string& line) { 
int VirtualAsciiFile::append_to_disk(const std::string& file_name) {

  // dump to disk the contents of the virtualasciifile, appending at the end of the given file_name

  int result = 0;

  if (_file_lines->size() > 0) {

    fstream output_file;

    output_file.open(file_name, ios::app);

    if (output_file.is_open()) {
@@ -310,6 +312,7 @@ int VirtualAsciiFile::append_to_disk(const std::string& file_name) { 
    } else {

      result = 1;

    }

  }

  return result;

}
@@ -334,6 +337,7 @@ int VirtualAsciiFile::insert(int32_t position, VirtualAsciiFile& rhs, int32_t st 
int VirtualAsciiFile::write_to_disk(const std::string& file_name) {

  // dump to disk the contents of the virtualasciifile, replacing the given file_name

  int result = 0;

  if (_file_lines->size() > 0) {

    fstream output_file;

    output_file.open(file_name, ios::out);

    if (output_file.is_open()) {
@@ -344,6 +348,7 @@ int VirtualAsciiFile::write_to_disk(const std::string& file_name) { 
    } else {

      result = 1;

    }

  }

  return result;

}
@@ -354,6 +359,7 @@ void VirtualAsciiFile::mpisend(const mixMPI *mpidata) { 
  int32_t num_lines =  _file_lines->size();

  MPI_Send(&num_lines, 1, MPI_INT32_T, 0, 10, MPI_COMM_WORLD);

  // now loop over data to send

  if (num_lines>0) {

    int32_t mysize;

    for (vector<string>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {

      // send the length of each string
@@ -363,6 +369,7 @@ void VirtualAsciiFile::mpisend(const mixMPI *mpidata) { 
      MPI_Send(it->c_str(), mysize+1, MPI_CHAR, 0, 10, MPI_COMM_WORLD);

    }

  }

}

#endif



/* >>> End of VirtualAsciiFile class implementation <<< */
@@ -483,9 +490,10 @@ VirtualBinaryFile::~VirtualBinaryFile() { 
  // for (vector<VirtualBinaryLine>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {

  //   delete it;

  // }

  while (!_file_lines->size() > 0) {

    _file_lines->pop_back();

  }

  _file_lines->clear();

  // while (_file_lines->size() > 0) {

  //   _file_lines->pop_back();

  // }

  if (_file_lines != NULL) delete _file_lines;

}
@@ -506,6 +514,7 @@ void VirtualBinaryFile::append_line(const VirtualBinaryLine& line) { 
int VirtualBinaryFile::append_to_disk(const std::string& file_name) {

  // dump to disk the contents of the virtualasciifile, appending at the end of the given file_name

  int result = 0;

  if (_file_lines->size() > 0) {

    fstream output_file;

    output_file.open(file_name, ios::app | ios::binary);

    if (output_file.is_open()) {
@@ -516,6 +525,7 @@ int VirtualBinaryFile::append_to_disk(const std::string& file_name) { 
    } else {

      result = 1;

    }

  }

  return result;

}
@@ -541,6 +551,7 @@ int VirtualBinaryFile::append_to_disk(const std::string& file_name) { 
int VirtualBinaryFile::write_to_disk(const std::string& file_name) {

  // dump to disk the contents of the virtualasciifile, replacing the given file_name

  int result = 0;

  if (_file_lines->size() > 0) {

    fstream output_file;

    output_file.open(file_name, ios::out | ios::binary);

    if (output_file.is_open()) {
@@ -551,6 +562,7 @@ int VirtualBinaryFile::write_to_disk(const std::string& file_name) { 
    } else {

      result = 1;

    }

  }

  return result;

}
@@ -561,10 +573,12 @@ void VirtualBinaryFile::mpisend(const mixMPI *mpidata) { 
  int32_t num_lines =  _file_lines->size();

  MPI_Send(&num_lines, 1, MPI_INT32_T, 0, 10, MPI_COMM_WORLD);

  // now loop over data to send

  if (num_lines>0) {

    for (vector<VirtualBinaryLine>::iterator it = _file_lines->begin(); it != _file_lines->end(); ++it) {

      it->mpisend(mpidata);

    }

  }

}

#endif



/* >>> End of VirtualBinaryFile class implementation <<< */