Fully Integrated CassiniIss In IsisImport (#4881) (535ae5a2) · Commits · aflab / astrogeology / ISIS3

isis/appdata/import/PDS3/CassiniISS.tpl

+28 −4

Original line number	Diff line number	Diff line
		@@ -27,7 +27,8 @@ InstrumentId = {{ INSTRUMENT_ID.Value }}
		TargetName = {{ targetName }}
		StartTime = {% set startTime=START_TIME.Value %}
		{{ RemoveStartTimeZ(startTime) }}
		StopTime = {{ STOP_TIME.Value }}
		StopTime = {% set stopTime=STOP_TIME.Value %}
		{{ RemoveStartTimeZ(stopTime) }}
		ExposureDuration = {{ EXPOSURE_DURATION.Value }} <Milliseconds>

		{% set antibloomingStateFlag = ANTIBLOOMING_STATE_FLAG.Value %}
		@@ -116,7 +117,8 @@ GainModeId = {{ gainModeId }} <ElectronsPerDN>
		{% set gainState = "0" %}
		{% endif %}
		GainState = {{ gainState }}
		ImageTime = {{ IMAGE_TIME.Value }}
		ImageTime = {% set imageTime=IMAGE_TIME.Value %}
		{{ RemoveStartTimeZ(imageTime) }}
		InstrumentDataRate = {{ INSTRUMENT_DATA_RATE.Value }} <KilobitsPerSecond>
		OpticsTemperature = ({{ OPTICS_TEMPERATURE.Value.0 }}, {{ OPTICS_TEMPERATURE.Value.1 }} <DegreesCelcius>)

		@@ -200,4 +202,26 @@ End_Group
		{% block translation %}
		CubeAtts = "+SignedWord+-32752:32767"
		DataPrefixBytes = {{ IMAGE.LINE_PREFIX_BYTES.Value }}
		StretchPairs = {{ CassiniIssStretchPairs() }}
		DataConversionType = {{ dataConversionType }}
		ValidMaximum = {{ VALID_MAXIMUM.Value.1 }}
		SummingMode = {{ summingMode }}
		CompressionType = {{ compressionType }}
		FlightSoftwareVersionId = {{ FLIGHT_SOFTWARE_VERSION_ID.Value }}
		VicarLabelBytes = {{ IMAGE_HEADER.BYTES.Value }}

		Object = AncillaryProcess
		ProcessFunction = cassiniIssCreateLinePrefixTable
		End_Object

		Object = AncillaryProcess
		ProcessFunction = cassiniIssFixLabel
		End_Object

		Object = PostProcess
		ProcessFunction = cassiniIssFixDnPostProcess
		StretchPairs = {{ CassiniIssStretchPairs() }}
		DataConversionType = {{ dataConversionType }}
		ValidMaximum = {{ VALID_MAXIMUM.Value.1 }}
		End_Object
		{% endblock %}

isis/src/base/apps/isisimport/CassiniImportUtils.h

0 → 100644

+252 −0

Original line number	Diff line number	Diff line
		#ifndef CASSINI_H
		#define CASSINI_H

		#include <vector>

		#include <QString>

		#include "EndianSwapper.h"
		#include "IsisImportBaseProcessFunctor.h"
		#include "ProcessByLine.h"
		#include "ProcessImport.h"
		#include "PvlGroup.h"
		#include "PvlKeyword.h"
		#include "Stretch.h"
		#include "Table.h"
		#include "TableField.h"
		#include "TableRecord.h"

		using namespace std;

		namespace Isis {

		/**
		* This class is used as a functor to perform post processing on cassini iss
		* images
		*
		* @author 2022-03-31 Adam Paquette
		*
		* @internal
		*/
		class CassiniIssFixDnFunctor : public ProcessFunctor {
		public:
		/**
		* Constructs a CassiniIssFunctor
		*
		* @param stats Pointer to a Statistics object to add data to
		* @param percent Sampling percentage of the image, used to calculate a line increment,
		* when calculating statistics
		*/
		CassiniIssFixDnFunctor(PvlKeyword &stretchPairs, QString dataConversionType, int validMax) {
		m_stretch = Stretch();
		for (size_t i = 0; i < stretchPairs.size(); i+=2) {
		m_stretch.AddPair(toDouble(stretchPairs[i]),
		toDouble(stretchPairs[i + 1]));
		}
		m_validMax = validMax;
		m_dataConversionType = dataConversionType;
		}

		virtual ~CassiniIssFixDnFunctor() {};

		void operator()(Buffer &buf) const {
		for(int i = 0; i < buf.size(); i++) {
		// zeros and negatives are valid DN values, according to scientists,
		// but likelyhood of a zero in 16 bit is rare,
		// so assume these are missing pixels and set them to null
		if(buf[i] == 0) {
		buf[i] = Null;
		}
		else if(m_dataConversionType == "Table") {
		buf[i] = m_stretch.Map((int)buf[i]);
		}
		// save max values (4095 for table-converted images and 255 for others) as HRS
		if(buf[i] >= m_validMax) {
		buf[i] = Hrs;
		}
		}
		};

		private:
		Stretch m_stretch; //!< Calculated stretch created from a stretch pair
		QString m_dataConversionType;
		int m_validMax; //!< Valid maximum as defined in the cassini iss image label
		};

		void cassiniIssFixDnPostProcess(QString ioFile, PvlObject postProcessObj) {
		PvlKeyword stretchPairsKeyword = postProcessObj["stretchPairs"];
		QString dataConversionType = postProcessObj["DataConversionType"];
		int validMax = toInt(postProcessObj["ValidMaximum"]);
		CassiniIssFixDnFunctor cassiniIssFixDnFunctor(stretchPairsKeyword, dataConversionType, validMax);

		ProcessByLine postProcess;
		CubeAttributeInput att;
		postProcess.SetInputCube(ioFile, att, ReadWrite);
		if(dataConversionType == "12Bit") {
		postProcess.Progress()->SetText("Setting special pixels and saving as 16bit...");
		}
		else if(dataConversionType == "8LSB") {
		postProcess.Progress()->SetText("Setting special pixels and saving as 16bit...");
		}
		//if ConversionType == Table, Use LUT to create stretch pairs for conversion
		else {
		postProcess.Progress()->SetText("Converting image pixels back to 12-bit and saving as 16bit...");
		}
		postProcess.StartProcess(cassiniIssFixDnFunctor);
		postProcess.EndProcess();
		};

		void cassiniIssCreateLinePrefixTable(Cube cube, vector<char > prefixData, PvlObject translation) {
		int sumMode = translation["SummingMode"];
		QString compressionType = translation["CompressionType"];
		double flightSoftware = 0.0;

		if (QString(translation["flightSoftwareVersionId"]).toStdString() != std::string("Unknown")) {
		flightSoftware = toDouble(translation["flightSoftwareVersionId"]);
		}

		PvlKeyword stretchPairsKeyword = translation["stretchPairs"];
		QString dataConversionType = translation["DataConversionType"];
		int validMax = toInt(translation["ValidMaximum"]);
		CassiniIssFixDnFunctor cassiniIssFixDnFunctor(stretchPairsKeyword, dataConversionType, validMax);

		TableField overclockPixels("OverclockPixels", TableField::Double, 3);
		//3 columns, first two are overclocked pixels and the third is their average
		TableRecord linePrefixRecord;
		linePrefixRecord += overclockPixels;
		Table linePrefixTable("ISS Prefix Pixels", linePrefixRecord);
		linePrefixTable.SetAssociation(Table::Lines);
		for(int l = 0; l < (int)prefixData.size(); l++) {
		unsigned char linePrefix = (unsigned char )(prefixData[l]);
		Buffer pixelBuf(1, 1, 1, SignedWord);

		vector<double> calibrationPixels;
		//Pixel data is MSB, see SIS version 1.1 page 17
		EndianSwapper swapper("MSB");

		vector<double> pixel;
		//12 is start byte for First Overclocked Pixel Sum in Binary Line Prefix, SIS version 1.1 page 94
		pixel.push_back(swapper.ShortInt(& (linePrefix[12])));
		//22 is start byte for Last Overclocked Pixel Sum in Binary Line Prefix, see SIS version 1.1 page 94
		pixel.push_back(swapper.ShortInt(& (linePrefix[22])));

		// section modelled after IDL CISSCAL's OverclockAvg() in cassimg_define.pro
		double overclockAvg = 0.0;
		if(compressionType != "Lossy" && flightSoftware < 1.3) { //numberOfOverclocks == 1
		// if Bltype CASSINI-ISS or CAS-ISS2, i.e. flight software version < 1.3
		// then there is only one column of valid overclocks in prefix pixels table,
		// the first column contains nulls, so use column 2 as average
		overclockAvg = pixel[1];
		}
		else { //numberOfOverclocks == 2
		// number of columns of valid overclocks in prefix pixels table is 2
		// for CAS-ISS3 or CAS-ISS4, i.e. flight software version 1.3 or 1.4
		// calculate appropriate average (as in cassimg_define.pro, CassImg::OverclockAvg())
		if(sumMode == 1) {
		overclockAvg = ((((double) pixel[0]) / 2 + ((double) pixel[1]) / 6) / 2);
		}
		if(sumMode == 2) {
		overclockAvg = ((((double)pixel[0]) + ((double) pixel[1]) / 3) / 2);
		}
		if(sumMode == 4) {
		overclockAvg = ((((double) pixel[0]) + ((double) pixel[1])) / 2);
		}
		else overclockAvg = 0;
		}
		// End Section

		pixel.push_back(overclockAvg);

		for(int i = 0; i < (int)pixel.size(); i++) {
		pixelBuf[0] = pixel[i];
		// Do 8 bit to 12 bit conversion for prefix data
		cassiniIssFixDnFunctor(pixelBuf);
		double pix = pixelBuf[0];
		if(pix == NULL8) {
		calibrationPixels.push_back(NULL2);
		}
		else if(pix == LOW_REPR_SAT8) {
		calibrationPixels.push_back(LOW_REPR_SAT2);
		}
		else if(pix == LOW_INSTR_SAT8) {
		calibrationPixels.push_back(LOW_INSTR_SAT2);
		}
		else if(pix == HIGH_INSTR_SAT8) {
		calibrationPixels.push_back(HIGH_INSTR_SAT2);
		}
		else if(pix == HIGH_REPR_SAT8) {
		calibrationPixels.push_back(HIGH_REPR_SAT2);
		}
		else {
		calibrationPixels.push_back(pix);
		}
		}
		linePrefixRecord[0] = calibrationPixels;
		linePrefixTable += linePrefixRecord;
		}
		cube->write(linePrefixTable);
		};

		void cassiniIssFixLabel(Cube cube, PvlObject translation, ProcessImport importer) {
		Pvl *outputLabel = cube->label();
		//Adjust Table-encoded values from 8 bit back to 12 bit.
		PvlKeyword stretchPairs = translation["stretchPairs"];
		Stretch stretch;
		for (size_t i = 0; i < stretchPairs.size(); i+=2) {
		stretch.AddPair(toDouble(stretchPairs[i]),
		toDouble(stretchPairs[i + 1]));
		}
		PvlGroup &inst = outputLabel->findGroup("Instrument", Pvl::Traverse);
		QString dataConversionType = translation["DataConversionType"];
		if(dataConversionType != "Table") { //Conversion Type is 12Bit or 8LSB, only save off overclocked pixels
		if(dataConversionType == "12Bit") {
		importer->Progress()->SetText("Image was 12 bit. No conversion needed. \nSaving line prefix data...");
		}
		else { //if (dataConversionType == "8LSB") {
		importer->Progress()->SetText("Image was truncated to 8 least significant bits. No conversion needed. \nSaving line prefix data...");
		}
		}
		else {
		double biasStripMean = inst.findKeyword("BiasStripMean");
		inst.findKeyword("BiasStripMean").setValue(toString(stretch.Map(biasStripMean)));
		inst.findKeyword("BiasStripMean").addComment("BiasStripMean value converted back to 12 bit.");
		importer->Progress()->SetText("Image was converted using 12-to-8 bit table. \nConverting prefix pixels back to 12 bit and saving line prefix data...");
		}

		unsigned char header = (unsigned char ) importer->FileHeader();
		int vicarLabelBytes = translation.findKeyword("VicarLabelBytes");
		int roo = (header + 50 + vicarLabelBytes) / 32 % 2; //*** THIS MAY NEED TO BE CHANGED,
		// SEE BOTTOM OF THIS FILE FOR IN DEPTH COMMENTS ON READOUTORDER
		inst.addKeyword(PvlKeyword("ReadoutOrder", toString(roo)));
		};
		}

		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
		// File header and readout order comments...
		// OUR FILE HEADER INCLUDES TWO SECTIONS:
		// -The first is the VICAR label (SIS page 52). The number of bytes included here is calculated in the IsisMain()
		// -The second is the Binary Label Header, or Binary Telemetry Header(SIS page 52). This contains 60 bytes (SIS page 84) of significant data.
		// The READOUT ORDER of an image is the order in which the cameras were read. This is needed for radiometric calibration (CISSCAL).
		// The possible values are :
		// 0 : Narrow-angle camera was read out first
		// 1 : Wide-angle camera was read out first
		// IDL CISSCAL FILE CASSIMG_SUBTRACTDARK.PRO LINE 333:
		// roo = bh[50]/32 MOD 2 ;Readout order is the 2nd bit of the 51st byte
		// According to SIS page 92 (Field=Software, Valid Values), the readout order is index 2 (the THIRD bit) of the byte.
		// Normally, we would assume that this was the third bit from the right, but there is some confusion on this matter.
		// SIS page 17 says bits and bytes are both "big endian" for pixel data, but doesn't mention whether this includes the binary telemetry table data,
		// Reading the first 3 bytes of the binary header and comparing with bit values described in SIS Table 7.3.2,
		// if the bytes are read as most significant bit first (left-to-right), each value matches up except summation mode.
		// In this case, SIS says they shoud be sum1:01, sum2:10, sum4:11. Actual values are sum1:00, sum2:01, sum4:10.
		// The IDL code also appears to be written as though bits are read in this manner, accessing the third bit from the left (32 ~ 00100000).
		// Since we haven't found a difinitive answer to this, we are mimicking the IDL code to determine the read out order.
		// We have not found an image with roo = 1 as of yet to test this.
		// If it is found to be the case that bits are read from left to right in this header, it may be more clear in the
		// future to rewrite the line using a logical bitwise &-operator: roo = *(header+50+vicarLabelBytes) & (00100000);
		// SOURCES :
		// Cassini ISS Tour VICAR Image Data File and Detatched PDS Label SIS, Tour Version 1.1 December 1, 2004
		// IDL cisscal application files: cassimg_subtractdark.pro and linetime.pro
		// -Jeannie Walldren 08/06/2008
		////////////////////////////////////////////////////////////////////////////////////////////////////////////////////////

		#endif

isis/src/base/apps/isisimport/importUtils.h

0 → 100644

+76 −0

Original line number	Diff line number	Diff line
		#ifndef IMPORT_UTILS_H
		#define IMPORT_UTILS_H

		#include <map>
		#include <iostream>
		#include <sstream>
		#include <vector>

		#include <QString>

		#include "Buffer.h"
		#include "EndianSwapper.h"
		#include "Table.h"
		#include "SpecialPixel.h"
		#include "Stretch.h"

		#include "CassiniImportUtils.h"

		using namespace std;

		std::map<std::string, int> processMap = {
		{"cassiniIssFixDnPostProcess", 1}
		};

		std::map<std::string, int> ancillaryProcessMap = {
		{"cassiniIssCreateLinePrefixTable", 1},
		{"cassiniIssFixLabel", 2}
		};

		namespace Isis {

		QString vectorToString(std::vector<double> v) {
		std::stringstream ss;
		ss << "(";

		size_t i = 0;
		for(; i < v.size(); i++)
		{
		if(i != 0)
		ss << ", ";
		ss << v[i];
		}
		ss << ")";
		return QString::fromStdString(ss.str());
		};

		void applyAncillaryProcess(Cube *cube,
		QString processFunction,
		PvlObject translation,
		ProcessImport *process) {
		vector<vector<char *>> prefixData = process->DataPrefix();
		switch (ancillaryProcessMap[processFunction.toStdString()]) {
		case 1:
		return cassiniIssCreateLinePrefixTable(cube,
		prefixData.at(0),
		translation);
		case 2:
		return cassiniIssFixLabel(cube, translation, process);
		}

		throw IException(IException::Programmer, "Unable to find prefix/suffix function [" + processFunction.toStdString() + "]", _FILEINFO_);
		};

		void runProcess(QString ioFile, PvlObject process) {
		std::string functionString = QString(process["ProcessFunction"]).toStdString();
		switch (processMap[functionString]) {
		case 1:
		cassiniIssFixDnPostProcess(ioFile, process);
		return;
		}

		throw IException(IException::Programmer, "Unable to find functor [" + functionString + "]", _FILEINFO_);
		};
		}

		#endif

isis/src/base/apps/isisimport/isisimport.cpp

+65 −6

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@

		#include "CubeAttribute.h"
		#include "FileName.h"
		#include "importUtils.h"
		#include "iTime.h"
		#include "OriginalLabel.h"
		#include "OriginalXmlLabel.h"
		@@ -24,8 +25,6 @@ using json = nlohmann::json;

		namespace Isis {



		void isisimport(UserInterface &ui, Pvl *log) {
		FileName fileTemplate = ("$ISISROOT/appdata/import/fileTemplate.tpl");
		json jsonData;
		@@ -148,6 +147,50 @@ namespace Isis {
		return bandInfo;
		});

		env.add_callback("CassiniIssStretchPairs", 0, [](Arguments& args) {
		PvlGroup &dataDir = Preference::Preferences().findGroup("DataDirectory");
		QString missionDir = (QString) dataDir["Cassini"];
		FileName *lutFile = new FileName(missionDir + "/calibration/lut/lut.tab");
		TextFile *stretchPairs = new TextFile(lutFile->expanded());

		vector<double> vectorStretchPairs = {};

		bool begindataFound = false;
		QString line;
		bool goodLine = stretchPairs->GetLine(line);

		// Search for tag "\begindata" if it was not already found by recursively using this method
		while (!begindataFound) {

		if (!goodLine) {
		// Might want to detail this probelm a little more
		return std::string("()");
		}

		if(!line.contains("\\begindata")) {
		goodLine = stretchPairs->GetLine(line);
		}
		else {
		begindataFound = true;
		}
		}

		// Create the stretch pairs
		double temp1 = 0;
		for(int i = 0; i < stretchPairs->LineCount(); i++) {
		stretchPairs->GetLine(line); //assigns value to line
		line = line.simplified();

		for (QString value: line.split(QRegExp("[\\s,]"), QString::SkipEmptyParts)) {
		vectorStretchPairs.push_back(temp1);
		vectorStretchPairs.push_back(toDouble(value));
		temp1++;
		}
		}

		return vectorToString(vectorStretchPairs).toStdString();
		});

		env.add_callback("splitOnChar", 2, [](Arguments& args){
		std::string text = args.at(0)->get<string>();

		@@ -250,7 +293,6 @@ namespace Isis {

		/**
		* Add SubFrame keyword to Instrument Group based on substring of ImageNumber.
		*
		*/
		env.add_callback("SetSubFrame", 1, [](Arguments& args) {
		std::string imageNumber = args.at(0)->get<string>();
		@@ -277,7 +319,6 @@ namespace Isis {

		/**
		* Remove units from keyword value if exists at the end of the string.
		*
		*/
		env.add_callback("RemoveUnits", 1, [](Arguments& args){

		@@ -393,6 +434,7 @@ namespace Isis {
		recSize = toInt(translation["DataFileRecordBytes"]);
		}
		importer.SetFileHeaderBytes(offset * recSize);
		importer.SaveFileHeader();
		}
		// Assume PDS4
		else {
		@@ -492,8 +534,8 @@ namespace Isis {
		if (translation.hasKeyword("CubeAtts")) {
		cubeAtts = QString(translation["CubeAtts"]);
		}
		CubeAttributeOutput att = CubeAttributeOutput(cubeAtts);
		Cube *outputCube = importer.SetOutputCube(ui.GetCubeName("TO"), att);
		CubeAttributeOutput outputAtts = CubeAttributeOutput(cubeAtts);
		Cube *outputCube = importer.SetOutputCube(ui.GetCubeName("TO"), outputAtts);

		if (isPDS4) {
		OriginalXmlLabel xmlLabel;
		@@ -516,8 +558,25 @@ namespace Isis {
		for(int g = 0; g < newCubeLabel.groups(); g++) {
		outCubeLabel.addGroup(newCubeLabel.group(g));
		}

		if (translation.hasObject("AncillaryProcess")) {
		for(int i = 0; i < translation.objects(); ++i) {
		PvlObject &object = translation.object(i);
		QString objectName = object.name();
		if (objectName == "AncillaryProcess") {
		applyAncillaryProcess(outputCube, QString(object["ProcessFunction"]), translation, &importer);
		}
		}
		}

		importer.EndProcess();

		if (translation.hasObject("PostProcess")) {
		// Expand for potentially more than one prefix process
		// See AncillaryProcess section above
		runProcess(ui.GetCubeName("TO"), translation.findObject("PostProcess"));
		}

		return;
		}
		}

isis/tests/FunctionalTestsIsisImport.cpp

+2 −2

Original line number	Diff line number	Diff line
		@@ -6,10 +6,12 @@
		#include <nlohmann/json.hpp>

		#include "Fixtures.h"
		#include "Histogram.h"
		#include "md5wrapper.h"
		#include "Pvl.h"
		#include "PvlGroup.h"
		#include "PvlKeyword.h"
		#include "TestUtilities.h"

		#include "isisimport.h"

		@@ -231,5 +233,3 @@ End)";

		EXPECT_EQ(archiveGroup["ObservationId"][0].toStdString(), "CRUS_000000_505_1");
		}