Commit 8709485b authored by Jesse Mapel's avatar Jesse Mapel
Browse files

Csmcsv (#4446) 

* CSM CSV output

* Small test clean-up

* typo fix

* obs header fix

* Remove param list from bo

* Added debug output

* Now reurning

* Fixed unit test

* xml save test update

* Clean up bundleresults unit test output

* Fixed qfile member

* Fixed old path

* Changed header to use param names

* Making separate csvs

* Added reverse on obs by instId

* Trying to fix multiple sensors

* Fixed images header

* More debug

* removed debug outs

* Added terrible imageindex search

* Added held config fix

* Updated for observation test

* Updated jigsaw test makefiles

* Updated with new tests
parent 861b04ff

isis/src/base/objs/CSMCamera/CSMCamera.cpp

+5 −5
Original line number Diff line number Diff line
@@ -864,7 +864,7 @@ namespace Isis { 
    for (int i = 0; i < paramList.size(); i++) {

      for (int j = 0; j < m_model->getNumParameters(); j++) {

        if (m_model->getParameterName(j) == paramList[i].toStdString()) {

          parameterIndices.push_back(i);

          parameterIndices.push_back(j);

        }

      }

    }

isis/src/control/apps/jigsaw/tsts/scconfig/Makefile

+8 −2
Original line number Diff line number Diff line
@@ -13,9 +13,15 @@ commands: 
	  bundleout_txt=no \

	  output_csv=no \

	  residuals_csv=no > /dev/null;

	$(CAT) bundleout_images.csv \

	$(CAT) bundleout_images_MARS_ODYSSEY_THEMIS_IR.csv \

	  | perl -pe 's/(^|,|: )([^,:]+\/)([^,\/:]*\.)(net|cub)/\1\3\4/g' 2> /dev/null \

	  > $(OUTPUT)/bundleout_scconfig_images.csv;

	  > $(OUTPUT)/bundleout_scconfig_images_themis.csv;

	$(CAT) bundleout_images_VIKING_ORBITER_1_VISUAL_IMAGING_SUBSYSTEM_CAMERA_B.csv \

	  | perl -pe 's/(^|,|: )([^,:]+\/)([^,\/:]*\.)(net|cub)/\1\3\4/g' 2> /dev/null \

	  > $(OUTPUT)/bundleout_scconfig_images_viking_1.csv;

	$(CAT) bundleout_images_VIKING_ORBITER_2_VISUAL_IMAGING_SUBSYSTEM_CAMERA_A.csv \

	  | perl -pe 's/(^|,|: )([^,:]+\/)([^,\/:]*\.)(net|cub)/\1\3\4/g' 2> /dev/null \

	  > $(OUTPUT)/bundleout_scconfig_images_viking_2.csv;

	$(RM) $(OUTPUT)/cubes.lis > /dev/null;

	$(RM) $(OUTPUT)/*.net > /dev/null;

	$(RM) *.csv > /dev/null;

isis/src/control/apps/jigsaw/tsts/scconfigHeld/Makefile

+12 −3
Original line number Diff line number Diff line
@@ -13,9 +13,18 @@ commands: 
	  bundleout_txt=no \

	  output_csv=no \

	  residuals_csv=no > /dev/null;

	$(CAT) bundleout_images.csv \

	$(CAT) bundleout_images_MARS_ODYSSEY_THEMIS_IR.csv \

	   | perl -pe 's/(^|,|: )([^,:]+\/)([^,\/:]*\.)(net|cub)/\1\3\4/g' 2> /dev/null \

	  > $(OUTPUT)/bundleout_scconfig_images.csv;

	   > $(OUTPUT)/bundleout_scconfig_images_themis.csv;

	$(CAT) bundleout_images_VIKING_ORBITER_1_VISUAL_IMAGING_SUBSYSTEM_CAMERA_B.csv \

	   | perl -pe 's/(^|,|: )([^,:]+\/)([^,\/:]*\.)(net|cub)/\1\3\4/g' 2> /dev/null \

	   > $(OUTPUT)/bundleout_scconfig_images_viking_1.csv;

	$(CAT) bundleout_images_VIKING_ORBITER_2_VISUAL_IMAGING_SUBSYSTEM_CAMERA_A.csv \

          | perl -pe 's/(^|,|: )([^,:]+\/)([^,\/:]*\.)(net|cub)/\1\3\4/g' 2> /dev/null \

          > $(OUTPUT)/bundleout_scconfig_images_viking_2.csv;

	$(CAT) bundleout_images_held.csv \

          | perl -pe 's/(^|,|: )([^,:]+\/)([^,\/:]*\.)(net|cub)/\1\3\4/g' 2> /dev/null \

          > $(OUTPUT)/bundleout_scconfig_images_held.csv;

	$(RM) $(OUTPUT)/cubes.lis > /dev/null;

	$(RM) $(OUTPUT)/hold.lis > /dev/null;

	$(RM) $(OUTPUT)/*.net > /dev/null;

isis/src/control/objs/BundleResults/unitTest.cpp

+10 −6
Original line number Diff line number Diff line
@@ -58,20 +58,19 @@ namespace Isis { 
      BundleResultsXmlHandlerTester(Project *project, XmlStackedHandlerReader *reader,

                                     FileName xmlFile) : BundleResults(project, reader) {



        QString xmlPath(xmlFile.expanded());

        QFile file(xmlPath);

        m_file.setFileName(xmlFile.expanded());



        if (!file.open(QFile::ReadOnly) ) {

        if (!m_file.open(QFile::ReadOnly) ) {

          throw IException(IException::Io,

                           QString("Unable to open xml file, [%1],  with read access").arg(xmlPath),

                           QString("Unable to open xml file, [%1],  with read access").arg(xmlFile.expanded()),

                           _FILEINFO_);

        }



        QXmlInputSource xmlInputSource(&file);

        QXmlInputSource xmlInputSource(&m_file);

        bool success = reader->parse(xmlInputSource);

        if (!success) {

          throw IException(IException::Unknown,

                           QString("Failed to parse xml file, [%1]").arg(xmlPath),

                           QString("Failed to parse xml file, [%1]").arg(m_file.fileName()),

                            _FILEINFO_);

        }
@@ -81,7 +80,12 @@ namespace Isis { 
       * Destroys the tester object

       */

      ~BundleResultsXmlHandlerTester() {

        if (m_file.exists()) {

          m_file.remove();

        }

      }



      QFile m_file;



  };

}

isis/src/control/objs/BundleSolutionInfo/BundleSolutionInfo.cpp

+85 −130
Original line number Diff line number Diff line
@@ -392,6 +392,7 @@ namespace Isis { 
  /**

   * @brief Outputs the header for the bundleout_images.csv file

   * @param fpOut The output file stream.

   * @param observation An observation for the instrument that this header is for.

   * @return True if the write is successful, False otherwise.

   *

   * @internal
@@ -399,7 +400,7 @@ namespace Isis { 
   *                           than the other headers. The number of TWIST headers will be the same

   *                           as each of the other angle headers. Fixes #4557.

   */

  bool BundleSolutionInfo::outputImagesCSVHeader(std::ofstream &fpOut) {

  bool BundleSolutionInfo::outputImagesCSVHeader(std::ofstream &fpOut, AbstractBundleObservationQsp observation) {



    if (!fpOut) {

      return false;
@@ -415,78 +416,11 @@ namespace Isis { 
    outputColumns.push_back("rms,");

    outputColumns.push_back("rms,");



    BundleObservationSolveSettings obsSettings = m_settings->observationSolveSettings(0);

    QStringList observationParameters = observation->parameterList();



    int numberCamPosCoefSolved = obsSettings.numberCameraPositionCoefficientsSolved();

    int numberCamAngleCoefSolved  = obsSettings.numberCameraAngleCoefficientsSolved();



    int nCoeff = 1;

    if (numberCamPosCoefSolved > 0)

      nCoeff = numberCamPosCoefSolved;



    for (int i = 0; i < nCoeff; i++) {

      for (int j = 0; j < 5; j++) {

        if (nCoeff == 1)

          outputColumns.push_back("X,");

        else {

          QString str = "X(t" + toString(i) + "),";

          outputColumns.push_back(str);

        }

      }

    }

    for (int i = 0; i < nCoeff; i++) {

      for (int j = 0; j < 5; j++) {

        if (nCoeff == 1)

          outputColumns.push_back("Y,");

        else {

          QString str = "Y(t" + toString(i) + "),";

          outputColumns.push_back(str);

        }

      }

    }

    for (int i = 0; i < nCoeff; i++) {

    for (int i = 0; i < observationParameters.size(); i++) {

      for (int j = 0; j < 5; j++) {

        if (nCoeff == 1) {

          outputColumns.push_back("Z,");

        }

        else {

          QString str = "Z(t" + toString(i) + "),";

          outputColumns.push_back(str);

        }

      }

      if (!i)

        break;

    }



    for (int i = 0; i < numberCamAngleCoefSolved; i++) {

      for (int j = 0; j < 5; j++) {

        if (numberCamAngleCoefSolved == 1)

          outputColumns.push_back("RA,");

        else {

          QString str = "RA(t" + toString(i) + "),";

          outputColumns.push_back(str);

        }

      }

    }

    for (int i = 0; i < numberCamAngleCoefSolved; i++) {

      for (int j = 0; j < 5; j++) {

        if (numberCamAngleCoefSolved == 1)

          outputColumns.push_back("DEC,");

        else {

          QString str = "DEC(t" + toString(i) + "),";

          outputColumns.push_back(str);

        }

      }

    }

    for (int i = 0; i < numberCamAngleCoefSolved; i++) {

      for (int j = 0; j < 5; j++) {

        if (numberCamAngleCoefSolved == 1) {

          outputColumns.push_back("TWIST,");

        }

        else {

          QString str = "TWIST(t" + toString(i) + "),";

          outputColumns.push_back(str);

        }

        outputColumns.push_back(observationParameters[i] + ",");

      }

    }
@@ -501,23 +435,13 @@ namespace Isis { 
    fpOut << buf;



    outputColumns.clear();

    outputColumns.push_back("Filename,");



    outputColumns.push_back("Filename,");

    outputColumns.push_back("sample res,");

    outputColumns.push_back("line res,");

    outputColumns.push_back("total res,");



    // Initially account for X,Y,Z (3)

    int nparams = 3;

    // See how many position coeffients we solved for to make more headers (t0, t1, ...)

    if (numberCamPosCoefSolved)

      nparams = 3 * numberCamPosCoefSolved;



    // Initially account for RA,DEC,TWIST (3)

    int numCameraAnglesSolved = 3;

    // See how many angle coefficients we solved for to make more headers (t0, t1, ...)

    nparams += numCameraAnglesSolved*numberCamAngleCoefSolved;

    for (int i = 0; i < nparams; i++) {

    for (int i = 0; i < observationParameters.size(); i++) {

      outputColumns.push_back("Initial,");

      outputColumns.push_back("Correction,");

      outputColumns.push_back("Final,");
@@ -1161,35 +1085,58 @@ namespace Isis { 
  bool BundleSolutionInfo::outputImagesCSV() {



    char buf[1056];

    int imgIndex = 0;



    QList<Statistics> rmsImageSampleResiduals = m_statisticsResults->rmsImageSampleResiduals();

    QList<Statistics> rmsImageLineResiduals = m_statisticsResults->rmsImageLineResiduals();

    QList<Statistics> rmsImageResiduals = m_statisticsResults->rmsImageResiduals();



    bool errorProp = false;

    if (m_statisticsResults->converged() && m_settings->errorPropagation()) {

      errorProp = true;

    }



    QList<QString> outputCsvFileNames;

    QList<QString> instrumentIds = m_statisticsResults->observations().instrumentIds();

    // If there's just a single instrumentId just call it bundleout_images.csv

    if (instrumentIds.size() == 1) {

      QString ofname = "bundleout_images.csv";

      ofname = m_settings->outputFilePrefix() + ofname;

      m_csvSavedImagesFilename = ofname;

      outputCsvFileNames.push_back(ofname);

    }

    // Otherwise append the instrument IDs so it's bundleout_images_spacecraft_sensor.csv

    else {

      for (int i = 0; i < instrumentIds.size(); i++) {

        QString updatedInstrumentId = instrumentIds[i];

        // Replace and "/" or " " characters with "_" to make the filename safer

        // This line must be separate to avoid modifying the instrumentId in the list

        // we will iterate over later

        updatedInstrumentId.replace("/", "_").replace(" ", "_");

        QString ofname = "bundleout_images_" + updatedInstrumentId + ".csv";

        ofname = m_settings->outputFilePrefix() + ofname;

        m_csvSavedImagesFilename = ofname;

        outputCsvFileNames.push_back(ofname);

      }

    }



    std::ofstream fpOut(ofname.toLatin1().data(), std::ios::out);

    for (int i = 0; i < instrumentIds.size(); i++) {



      std::ofstream fpOut(outputCsvFileNames[i].toLatin1().data(), std::ios::out);

      if (!fpOut) {

        return false;

      }



      QList<AbstractBundleObservationQsp> observations =

          m_statisticsResults->observations().observationsByInstId(instrumentIds[i]);



    AbstractBundleObservationQsp observation;

      int nObservations = observations.size();



    int nObservations = m_statisticsResults->observations().size();

      outputImagesCSVHeader(fpOut, observations.front());



    outputImagesCSVHeader(fpOut);



    bool errorProp = false;

    if (m_statisticsResults->converged() && m_settings->errorPropagation()) {

      errorProp = true;

    }



    for (int i = 0; i < nObservations;i++ ) {

      observation = m_statisticsResults->observations().at(i);

      for (int j = 0; j < nObservations; j++ ) {

        AbstractBundleObservationQsp observation = observations[j];



        if(!observation) {

          continue;
@@ -1197,10 +1144,17 @@ namespace Isis { 


        int numImages = observation->size();



      for (int j = 0; j < numImages; j++) {

        for (int k = 0; k < numImages; k++) {



        BundleImageQsp image = observation->at(j);

          BundleImageQsp image = observation->at(k);



          // TODO this is a bad linear search, can we change the rms vectors to maps?

          int observationIndex = m_statisticsResults->observations().indexOf(observation);

          int imgIndex = k; // Start at k to account for the images in this observation

          // Accumulate all of the images from previous observations

          for (int obsIndex = 0; obsIndex < observationIndex; obsIndex++) {

            imgIndex += m_statisticsResults->observations().at(obsIndex)->size();

          }



          sprintf(buf, "%s", image->fileName().toLatin1().data());

          fpOut << buf;
@@ -1235,8 +1189,9 @@ namespace Isis { 


        }

      }



      fpOut.close();

    }



    return true;

  }