Force hical to fall back to ZeroDark if ZeroDarkRate fails on init (#4986) (5dc835e1) · Commits · aflab / astrogeology / ISIS3

isis/src/mro/apps/hical/hical.cpp

+50 −25

Original line number	Diff line number	Diff line
		@@ -172,30 +172,11 @@ namespace Isis {
		ZrHist.add("Debug::SkipModule invoked!");
		}

		/////////////////////////////////////////////////////////////////
		// ZeroDark removes dark current
		//
		procStep = "ZeroDark module";
		hiconf.selectProfile("ZeroDark");
		hiprof = hiconf.getMatrixProfile();
		HiHistory ZdHist;
		ZdHist.add("Profile["+ hiprof.Name()+"]");
		if ( !SkipModule(hiprof) ) {
		ZeroDark zd(hiconf);
		calVars->add(hiconf.getProfileName(), zd.ref());
		ZdHist = zd.History();
		if ( hiprof.exists("DumpModuleFile") ) {
		zd.Dump(hiconf.getMatrixSource("DumpModuleFile",hiprof));
		}
		}
		else {
		calVars->add(hiconf.getProfileName(), HiVector(nsamps, 0.0));
		ZdHist.add("Debug::SkipModule invoked!");
		}

		/////////////////////////////////////////////////////////////////
		// ZeroDarkRate removes dark current with a new approach compared
		// with ZeroDark.
		bool zdrFallback = false;
		procStep = "ZeroDarkRate module";
		HiHistory ZdrHist;
		ZdrHist.add("Profile[ZeroDarkRate]");
		@@ -207,6 +188,7 @@ namespace Isis {
		else {
		hiconf.selectProfile("ZeroDarkRate");
		hiprof = hiconf.getMatrixProfile();

		if ( !SkipModule(hiprof) ) {
		// Make sure we aren't applying ZeroDark and ZeroDarkRate
		if ( !SkipModule(hiconf.getMatrixProfile("ZeroDark")) ){
		@@ -217,6 +199,7 @@ namespace Isis {
		"in your configuration file.";
		throw IException(IException::User, mess, _FILEINFO_);
		}
		try{
		ZeroDarkRate zdr(hiconf);
		calVars->add(hiconf.getProfileName(), zdr.ref());
		ZdrHist = zdr.History();
		@@ -224,12 +207,54 @@ namespace Isis {
		zdr.Dump(hiconf.getMatrixSource("DumpModuleFile",hiprof));
		}
		}
		catch(IException e){
		if (hiprof.exists("Fallback") && IsTrueValue(hiprof, "Fallback")){
		zdrFallback = true;
		calVars->add(hiconf.getProfileName(), HiVector(nsamps, 0.0));
		std::cerr << "Falling back to ZeroDark implementation. Unable to initialize ZeroDarkRate "
		<< "module with the following error:" << std::endl << e.what() << "\nContinuing..."<< std::endl;
		ZdrHist.add("Debug::Unable to initialize ZeroDarkRate module. "
		"Falling back to ZeroDark implementation");
		} else {
		// If fallback is not found or fallback is false, throw the exception
		std::cerr<< "\nNot all combinations of CCD, channel, TDI rate, binning value, and ADC setting "
		"have a DarkRate*.csv available, and you may need to run hical with ZeroDark instead "
		"of ZeroDarkRate specified in the configuration file. Alternatively, you may specify "
		"Fallback = True in the ZeroDarkRate configuration profile to automatically use the "
		"ZeroDark module on ZeroDarkRate failure.\n" << std::endl;
		throw(e);
		}
		}
		}
		else {
		calVars->add(hiconf.getProfileName(), HiVector(nsamps, 0.0));
		ZdrHist.add("Debug::SkipModule invoked!");
		}
		}

		/////////////////////////////////////////////////////////////////
		// ZeroDark removes dark current
		//
		procStep = "ZeroDark module";
		hiconf.selectProfile("ZeroDark");
		hiprof = hiconf.getMatrixProfile();
		HiHistory ZdHist;
		ZdHist.add("Profile["+ hiprof.Name()+"]");
		// If ZeroDark module is enabled or we need to fall back to zero dark from zero dark rate
		if ( !SkipModule(hiprof) \|\| zdrFallback) {
		ZeroDark zd(hiconf);
		calVars->add(hiconf.getProfileName(), zd.ref());
		ZdHist = zd.History();
		if ( hiprof.exists("DumpModuleFile") ) {
		zd.Dump(hiconf.getMatrixSource("DumpModuleFile",hiprof));
		}
		}
		else {
		calVars->add(hiconf.getProfileName(), HiVector(nsamps, 0.0));
		ZdHist.add("Debug::SkipModule invoked!");
		}


		////////////////////////////////////////////////////////////////////
		// GainLineDrift correct for gain-based drift
		//

isis/tests/FunctionalTestsHical.cpp

+24 −0

Original line number	Diff line number	Diff line
		@@ -534,3 +534,27 @@ TEST(HicalTest, DarkRate) {
		parameterValues = patameterMatches.next();
		EXPECT_NEAR(parameterValues.captured(0).toDouble(), 163.44824721503, 0.00001);
		}

		TEST(HicalTest, DarkRateFallback) {
		QTemporaryDir prefix;
		QString outFileName = prefix.path() + "/out.cub";
		QVector<QString> args = { "FROM=data/hical/mroHical.cub",
		"TO=" + outFileName,
		"CONF=data/hical/hical.0023_darkrate_missing.conf",
		"OPATH=" + prefix.path() + "/" };
		UserInterface options(APP_XML, args);

		try {
		hical(options);
		}
		catch (IException &e) {
		FAIL() << e.toString().toStdString().c_str() << std::endl;
		}

		// Check calibrated cube
		Cube outCube(outFileName);

		std::unique_ptr<Statistics> stats (outCube.statistics());
		EXPECT_NEAR(stats->Average(), 0.066949089371337325, .00001);
		EXPECT_NEAR(stats->StandardDeviation(), 0.004873520482354521, .00001);
		}
		No newline at end of file

isis/tests/data/hical/hical.0023_darkrate_missing.conf

0 → 100644

+296 −0

Original line number	Diff line number	Diff line
		# HiRISE Calibration Matricies configuration file
		# See documentation for the hical application on the content and form of
		# this file.
		# $Id$
		Object = Hical

		Program = "hical"

		Name = "HiMatrices"
		DefaultProfile = "HiMatrices"

		/* If you want to rerun hical, you must set PropagateTables to True. Use */
		/* this in conjuction with Debug::SkipModule = True option for each module. */
		PropagateTables = True

		/* Define label groups that are loaded for each profile reference */
		/* Note all keywords in these groups become available to all profiles. */
		/* Thus, you can use them in the OptionKeywords and ProfileOptions keywords */
		/* to create very specialized profiles for special needs. */
		/* Specify the FPA reference temperature. It is used in several modules so */
		/* it is specified at the top level */

		LabelGroups = ( "Dimensions", "Instrument", "Archive")

		/* These keywords are used in ProfileOptions mapping. Note that order and */
		/* case matter! WARNING: You can easily break file lookups if these keys */
		/* are deleted or modified improperly!!! */
		OptionKeywords = ("FILTER", "CCD", "CHANNEL", "TDI", "BIN", "ProductId",
		"Program", "Module", "OPATH", "CalOptions")

		/* Additional profile combinations and order load hierarchy. These keywords */
		/* are defined when the LabelGroups are loaded. */
		/* Kris Becker & Eric Eliason updated 10/24/2008 */
		/* ProfileOptions value added: {Module}_{CalOptions} */
		ProfileOptions = ("{FILTER}", "TDI{TDI}", "BIN{BIN}", "TDI{TDI}/BIN{BIN}",
		"{FILTER}{CCD}_{CHANNEL}",
		"{FILTER}{CCD}_{CHANNEL}/TDI{TDI}/BIN{BIN}", "Debug",
		"{Module}_{CalOptions}")

		/* Specify the FPA reference temperature. It is used in several modules so */
		/* it is specified at the top level */
		FpaReferenceTemperature = 21.0

		/* This profile contains parameters pertinent to processing the buffer */
		/* pixels for subsequent using in the down-line offset correction, ZeroBufferFit module */
		Group = Profile
		Name = ZeroBufferSmooth
		Module = ZeroBufferSmooth
		ZeroBufferSmoothFirstSample = 5
		ZeroBufferSmoothLastSample = 11

		ZeroBufferSmoothFilterWidth = 201
		ZeroBufferSmoothFilterIterations = 2
		End_Group

		/* This profile contains parameters pertinent to processing the down-line offset correction */
		/* We do not use this by default because a functional fit misses important bumps and wiggles in */
		/* the buffer pixels, which are required to remove similar noise from the image area. */

		Group = Profile
		Name = ZeroBufferFit
		Module = ZeroBufferFit

		/* Uncomment to turn off non linear fitting of ZeroBufferSmooth data and pass it thru */
		ZeroBufferFitSkipFit = True

		/* Uncomment to use linear fitting of ZeroBufferSmooth data when non-linear fails */
		/* Default is to pass thru filtered ZeroBufferSmooth data */
		/* ZeroBufferFitOnFailUseLinear = True */

		/* Minimum number of good lines (NLines - (TrimLines/Summing)) to fit */
		ZeroBufferFitMinimumLines = 250

		/* Maximum number of iterations for the algorithm to converge and */
		/* other limits */
		MaximumIterations = 100
		MaximumLog = 709.0

		/* Convergence parameters for Levenberg-Marquardt algorithm */
		/* Equation is solved when \|dx_i\| < AbsoluteError + RelativeError * \|x_i\| */
		/* where dx is the last step and x is the current step for each i-th */
		/* value */
		AbsoluteError = 1.0E-4
		RelativeError = 1.0E-4

		/* Filtering of the guestimate buffer */
		GuessFilterWidth = 17
		GuessFilterIterations = 1

		# DumpModuleFile = "{ProductId}_{Module}.log"
		End_Group

		/* This profile contains parameters pertinent to processing the offset in the columns */
		Group = Profile
		Name = ZeroReverse
		Module = ZeroReverse

		/* Set calibration parameters for hiclean operations. Indexes are all 0-based */
		ZeroReverseFirstLine = 1
		ZeroReverseLastLine = 19


		/* Reverse Clock trigger Statistics profiles */
		ReverseClockStatistics = "$mro/calibration/matrices/ReverseClockStatistics.????.conf"
		RevLisTolerance = 1
		RevHisTolerance = 1
		RevNulTolerance = 1
		End_Group

		/* Skip reverse clock if the ReverseReadoutNoise is to large */
		/* Profile added by Kris Becker & Eric Eliason, 10/25/2008 */
		Group = Profile
		Name = Zz_ReverseReadoutNoise
		End_Group

		/* This profile contains parameters pertinent to processing dark current. */
		/* Required label keywords: Summing, Tdi, FpaPositiveYTemperature, */
		/* and FpaNegativeYTemperature, Lines */
		/* Also needs LineTime which is computed. */
		Group = Profile
		Debug::SkipModule = True
		Name = ZeroDark
		Module = ZeroDark

		/* Define the B matrix file reference */
		/* As of version 0020, 9 March 2010, we will be using the following names and formats */
		/* We are calling this the DarkCurrent now. The filename will stay the same (with a .csv extension) */
		DarkCurrent = "$mro/calibration/matrices/B_TDI{TDI}_BIN{BIN}_hical_????.csv"
		DarkCurrentColumnName = "{CCD}/{CHANNEL}"

		/* The slope and intercepts to the temperature-dependent correction to the dark current are given below. */
		DarkSlope = "$mro/calibration/matrices/B_Temperature_Slope_hical_????.csv"
		DarkSlopeColumnName = "CH{CHANNEL}_TDI{TDI}"

		DarkIntercept = "$mro/calibration/matrices/B_Temperature_Intercept_hical_????.csv"
		DarkInterceptColumnName = "CH{CHANNEL}_TDI{TDI}"

		/* Do filtering? */
		ZeroDarkFilterWidth = 3
		ZeroDarkFilterIterations = 1
		End_Group

		/* This profile contains parameters pertinent to processing dark current based on temperature. */
		/* Required label keywords: Summing, Filter, Tdi, Ccd, Channel, FpaPositiveYTemperature, */
		/* FpaNegativeYTemperature, Lines */
		Group = Profile
		Name = ZeroDarkRate
		Module = ZeroDarkRate

		/* Define the DarkRate matrix file reference */
		DarkRate = "missing.csv"
		Fallback = True
		End_Group

		/* This profile contains parameters pertinent to processing the nonlinear gain correction */
		Group = Profile
		Name = GainNonLinearity
		Module = GainNonLinearity

		/* Define the nonlinearity correction coefficients */
		NonLinearityGain = "$mro/calibration/matrices/Gain_NonLinearity_BIN{BIN}_hical_????.csv"
		NonLinearityGainRowName = "{CCD}_{CHANNEL}"
		End_Group



		/* This profile contains parameters pertinent to processing down-line gain correction. */
		/* Required label keywords: CpmmNumber, ChannelNumber, Lines */
		/* This module has been reactivated as of 2010-01-29 */
		Group = Profile
		Name = GainLineDrift
		Module = GainLineDrift

		SkipLines = 1
		/* Added "_hicalbeta" to filename as of 2008-04-02. This is consistant with */
		/* naming convention used for the beta version of hical. */
		GainLineCoefficients = "$mro/calibration/matrices/line_correct_{BIN}_hical_????.csv"

		/* As of version 0020, 9 March 2010, these are more correctly, called the LineGainDrift correction*/
		LineGainDrift = "$mro/calibration/matrices/Line_Gain_Drift_BIN{BIN}_hical_????.csv"
		LineGainDriftColumnHeader = True
		LineGainDriftRowName = "{CCD}/{CHANNEL}"
		End_Group

		/* This profile contains parameters pertinent to processing gain correction. */
		/* Required label keywords: CpmmNumber, ChannelNumber, Lines */
		Group = Profile
		Name = GainChannelNormalize
		Module = GainChannelNormalize

		/* Define the G matrix file reference */
		/* As of version 0020, 9 March 2010, these coefficients have been renamed to Gains. */
		Gains = "$mro/calibration/matrices/Gains_hical_????.csv"
		GainsRowName = "{BIN}"
		GainsColumnName = "{CCD}/{CHANNEL}"
		End_Group

		/* This profile contains parameters pertinent to processing gain correction. */
		/* Required label keywords: CpmmNumber, ChannelNumber, Tdi, Lines */
		Group = Profile
		Name = GainFlatField
		Module = GainFlatField

		/* Define the A matrix file reference */
		/* As of version 0020, 9 March 2010, These have the same filename with a .csv extension.*/
		/* As of version 0020, 9 March 2010, These coefficients are now called Flats. */
		Flats = "$mro/calibration/matrices/A_TDI{TDI}_BIN{BIN}_hical_????.csv"
		FlatsColumnName = "{CCD}/{CHANNEL}"
		End_Group

		/* This profile contains parameters pertinent to processing */
		/* temperature-dependant gain correction. Formally in GainFlatField module. */
		/* Required label keywords: CpmmNumber, ChannelNumber, Samples */
		Group = Profile
		Name = GainTemperature
		Module = GainTemperature

		/* Define temperature-dependant gain correction CSV file */
		FpaTemperatureFactorSkipLines = 4
		FpaTemperatureFactorHeader = True
		FpaTemperatureFactorFile = "$mro/calibration/matrices/FpaTemperatureGain_BIN{BIN}.????.csv"

		/* As of version 0020, 9 March 2010, This is now called the FPAGain. */
		FPAGain = "$mro/calibration/matrices/Temperature_Gain_????.csv"
		FPAGainColumnName = "{CCD}/{CHANNEL}"
		FPAGainRowName = "{BIN}"

		End_Group

		/* This profile contains parameters pertinent to processing I/F conversion. */
		/* Required label keywords: ScanExposureDuration */
		Group = Profile
		Name = GainUnitConversion
		Module = GainUnitConversion

		/* I/F correction for tdi/bin - currently set at 1.0 for all tdi/bin */h
		/* combinations. */
		GainUnitConversionBinFactor = 1.0
		End_Group

		/* Here are the filter profiles. All keywords that pertain to a filter set */
		/* should be specified here. FilterGainCorrection are I/F corrections in */
		/* units of DN/s. */
		/* Added CalFact, CCD QE, Temp dependence correction 2010-04-28 */
		Group = Profile
		Name = RED
		FilterGainCorrection = 157702564.0
		IoverFbasetemperature = 18.9
		QEpercentincreaseperC = 0.0005704
		AbsGain_TDI128 = 6.376583
		End_Group

		Group = Profile
		Name = IR
		FilterGainCorrection = 56464791.0
		IoverFbasetemperature = 18.9
		QEpercentincreaseperC = 0.002696
		AbsGain_TDI128 = 6.989840
		End_Group

		Group = Profile
		Name = BG
		FilterGainCorrection = 115074166.0
		IoverFbasetemperature = 18.9
		QEpercentincreaseperC = 0.00002295
		AbsGain_TDI128 = 6.997557
		End_Group

		Group = Profile
		Name = IR10_1
		# LastGoodLine = 3100
		End_Group


		Group = Profile
		Name = Debug

		/ Current disables writting to label history due to bug in keyword formatter in ISIS /
		/* The bug has the following error signature: */
		/* terminate called after throwing an instance of 'std::out_of_range' */
		/* what(): basic_string::substr */
		/* Abort */
		/* You must set this to false when this occurs as a workaround and use the */
		/* DumpHistoryFile parameter to see the parameter history. */
		LogParameterHistory = False

		/* Uncomment this line to write parameter history to the ProductId log */
		DumpHistoryFile = "{OPATH}/{ProductId}.{Program}.log"

		/* Uncomment this line to dump Module data for every module when using Debug */
		/* profiling. */
		# DumpModuleFile = "{OPATH}/{ProductId}_{Module}.log"
		End_Group

		End_Object