Unverified Commit aafa3031 authored by gsn9's avatar gsn9 Committed by GitHub
Browse files

Pds mer mi test (#4959) 

* pushing what I have for now, need to finish the mermi test

* need to fix mermi tpl to pass test still

* updated template and test, note test is still failing

* fixed test & template

* added file for testing

* added case two for MerMI test and it's required IMG file, as well as removing full path from cropt_mer notebook

* removed full paths from IMG files
parent 50b7031d

isis/appdata/import/PDS3/MerMI.tpl

+6 −4
Original line number Diff line number Diff line
@@ -107,11 +107,13 @@ 
    {% else %}

    InstrumentTemperature          = NULL

    {% endif %}



    {% if exists("INSTRUMENT_STATE_PARMS.INSTRUMENT_TEMPERATURE_NAME") %}

    InstrumentTemperatureName      = ({{ join(INSTRUMENT_STATE_PARMS.INSTRUMENT_TEMPERATURE_NAME.Value, ",") }})

    InstrumentTemperatureName = ("{{ join(INSTRUMENT_STATE_PARMS.INSTRUMENT_TEMPERATURE_NAME.Value, "\", \"")}}")

    {% else %}

    InstrumentTemperatureName = NULL

    {% endif %}



    OffsetModeID                  = {% if exists("INSTRUMENT_STATE_PARMS.OFFSET_MODE_ID") %}

                                    {{ INSTRUMENT_STATE_PARMS.OFFSET_MODE_ID.Value }}

                                    {% else %}
@@ -122,8 +124,8 @@ 
                                    {% else %}

                                    NULL

                                    {% endif %}

    TemperatureMiCCD            = {{ instTemp.6 }}

    TemperatureMiElectronics    = {{ instTemp.7 }}

    TemperatureMiCCD            = {{ INSTRUMENT_STATE_PARMS.INSTRUMENT_TEMPERATURE.Value.6}}

    TemperatureMiElectronics    = {{ INSTRUMENT_STATE_PARMS.INSTRUMENT_TEMPERATURE.Value.7 }}

{% endblock %}



{% block additional_groups %}

isis/notebooks/crop_mer_mi.ipynb

0 → 100644
+167 −0
Original line number Diff line number Diff line 
%% Cell type:code id: tags:



``` python

import pvl

import struct

import matplotlib.pyplot as plt

import numpy as np

import datetime

import os.path

import binascii

```



%% Cell type:code id: tags:



``` python

mimap_file = 'path/to/input/file.IMG'

image_file = mimap_file

```



%% Cell type:code id: tags:



``` python

header = pvl.load(mimap_file)

header

```



%% Cell type:code id: tags:



``` python

with open(mimap_file, 'rb') as f:

    image_offset = ((header["^IMAGE"] -1) * (header["RECORD_BYTES"])) -1

    f.seek(image_offset)

    b_image_data = f.read()



print(b_image_data[0:20], image_offset)

```



%% Cell type:code id: tags:



``` python

n_lines = 4

n_samples = 5

line_length = header['IMAGE']['LINE_SAMPLES'] * (header['IMAGE']['SAMPLE_BITS']//8)

```



%% Cell type:code id: tags:



``` python

image_data = []

line_offset = 100

sample_offset = 100



for j in range(line_offset, n_lines * header['IMAGE']['BANDS'] + line_offset + 1):

    print(j*line_length,(j+1)*line_length)

    image_sample = np.frombuffer(b_image_data[j*line_length:(j+1)*line_length], dtype=np.int16, count=int(line_length/2))

    image_data.append(image_sample)

image_data = np.array(image_data)

image_data = image_data[:, sample_offset:sample_offset + n_samples]

print(image_data.shape)

```



%% Cell type:code id: tags:



``` python

plt.figure(figsize=(20,20))

plt.imshow(image_data,cmap="gray")#[0:20,0:20])

```



%% Cell type:code id: tags:



``` python

print(image_data[0][0])





class RealIsisCubeLabelEncoder(pvl.encoder.ISISEncoder):

    def encode_time(self, value):

        if value.microsecond:

            second = u'%02d.%06d' % (value.second, value.microsecond)

        else:

            second = u'%02d' % value.second

            time = u'%02d:%02d:%s' % (value.hour, value.minute, second)

        return time



```



%% Cell type:code id: tags:



``` python

import math



image_fn, image_ext = os.path.splitext(image_file)

crop = '_cropped'

mini_image_fn = image_fn + crop + image_ext

print(mini_image_fn)

mini_image_bn = os.path.basename(mini_image_fn)

grammar = pvl.grammar.ISISGrammar()

grammar.comments+=(("#", "\n"), )

encoder = RealIsisCubeLabelEncoder()

# Overwrite the number of lines in the label

header['IMAGE']['LINES'] = n_lines + 1

header['IMAGE']['LINE_SAMPLES'] = n_samples

actual_label_size = len(pvl.dumps(header, encoder=encoder, grammar=grammar)) / header["RECORD_BYTES"]

print(actual_label_size)

header['^IMAGE'] = math.ceil(actual_label_size) + 1

print(int(header["RECORD_BYTES"] * (math.ceil(actual_label_size) - actual_label_size)))

# - 1 from 0 based and another - 1 from the new line written before the padding

padding = ['0' for i in range(0, (int(header["RECORD_BYTES"] * (math.ceil(actual_label_size) - actual_label_size))) - 1)]

padding = ''.join(padding)

padding = padding.encode('utf8')

print(padding)

```



%% Cell type:code id: tags:



``` python

label_fn, label_ext = os.path.splitext(mimap_file)

out_label = label_fn + crop + label_ext



pvl.dump(header, out_label, encoder=encoder, grammar=grammar)

```



%% Cell type:code id: tags:



``` python

with open(mini_image_fn, 'ab+') as f:

    b_reduced_image_data = image_data.tobytes()

    f.write(b'\n')

    f.seek(0, 2)

    f.write(padding)

    f.write(b_reduced_image_data)

print(len(b_reduced_image_data),len(padding))

```



%% Cell type:code id: tags:



``` python

pvl.load(mini_image_fn)

```



%% Cell type:code id: tags:



``` python

with open(mini_image_fn, 'rb') as f:

  image_offset = ((header["^IMAGE"] -1) * header["RECORD_BYTES"])

  f.seek(image_offset)

  b_image_data = f.read()

```



%% Cell type:code id: tags:



``` python

print(image_offset,b_image_data,len(b_image_data))

image_data = np.frombuffer(b_image_data, dtype=np.int16).reshape((n_lines + 1, n_samples))

plt.figure(0, figsize=(20, 20))

plt.imshow(image_data)

plt.show()

```



%% Cell type:code id: tags:



``` python

```



%% Cell type:code id: tags:



``` python

```

isis/tests/FunctionalTestsIsisImportMerMI.cpp

0 → 100644
+271 −0
Original line number Diff line number Diff line 
#include <iostream>

#include <time.h>



#include <QRegExp>

#include <QString>

#include <nlohmann/json.hpp>



#include "Fixtures.h"

#include "md5wrapper.h"

#include "Pvl.h"

#include "PvlGroup.h"

#include "PvlKeyword.h"



#include "isisimport.h"

#include "TestUtilities.h"



#include "gmock/gmock.h"



using namespace Isis;

using json = nlohmann::json;



static QString APP_XML = FileName("$ISISROOT/bin/xml/isisimport.xml").expanded();





TEST_F(TempTestingFiles, FunctionalTestIsisImportMerMICaseOne){

    std::istringstream PvlInput(R"(

        Object = IsisCube

            Object = Core

                StartByte   = 65537

                Format      = Tile

                TileSamples = 128

                TileLines   = 128



                Group = Dimensions

                    Samples = 5

                    Lines   = 5

                    Bands   = 1

                End_Group



                Group = Pixels

                    Type       = SignedWord

                    ByteOrder  = Lsb

                    Base       = 0.0

                    Multiplier = 1.0

                End_Group

            End_Object



            Group = Instrument

                RoverMotionCounter          = (64, 472, 321, 1798, 1331)

                RoverMotionCounterName      = (SITE, DRIVE, IDD, PMA, HGA)

                SpacecraftName              = MARS_EXPLORATION_ROVER_1

                InstrumentID                = MI

                InstrumentName              = "MICROSCOPIC IMAGER"

                InstrumentSerialNumber      = 110

                LocalTrueSolarTime          = 12:07:38

                PlanetDayNumber             = 691

                SolarLongitude              = 350.592

                SpacecraftClockCntPartition = 1

                SpacecraftClockStartCount   = 189529263.621

                SpacecraftClockStopCount    = 189529263.774

                StartTime                   = 2006-01-03T02:59:18.555

                StopTime                    = 2006-01-03T02:59:18.707

                ExposureDuration            = 153.6 <ms>

                ExposureDurationCount       = 30

                FilterName                  = MI_OPEN

                FilterNumber                = 2

                FlatFieldCorrectionFlag     = FALSE

                InstrumentModeID            = FULL_FRAME

                InstrumentTemperature       = (-13.9083, -3.79407, 0.272201, -6.62645,

                                            -7.49551, -8.20222, -3.87095, -11.4134,

                                            0.0) <degC>

                InstrumentTemperatureName   = ("FRONT HAZ ELECTRONICS",

                                            "REAR HAZ ELECTRONICS",

                                            "LEFT PAN ELECTRONICS", "LEFT PAN CCD",

                                            "RIGHT PAN CCD", "LEFT NAV CCD", "MI CCD",

                                            "MI ELECTRONICS", "EDL CCD")

                OffsetModeID                = 4080

                ShutterEffectCorrectionFlag = TRUE

                TemperatureMiCCD            = -3.87095

                TemperatureMiElectronics    = -11.4134

            End_Group



            Group = Archive

                DataSetID   = MER1-M-MI-2-EDR-SCI-V1.0

                DataSetName = "MER 1 MARS MICROSCOPIC IMAGER SCIENCE EDR VERSION 1.0"

                ProductID   = 1M189529263EFF64KCP2977M2F1

            End_Group



            Group = MerImageRequestParms

                PixelAveragingHeight = 1

                PixelAveragingWidth  = 1

            End_Group



            Group = MerSubframeRequestParms

                FirstLine        = 1

                FirstLineSamples = 1

            End_Group

        End_Object



        Object = Label

            Bytes = 65536

        End_Object



        Object = History

            Name      = IsisCube

            StartByte = 2162689

            Bytes     = 473

        End_Object



        Object = OriginalLabel

            Name      = IsisCube

            StartByte = 2163162

            Bytes     = 19191

        End_Object

        End         

    )");

  QVector<QString> args = {"from=data/isisimport/1M189529263EFF64KCP2977M2F1_cropped.IMG", "to=" + tempDir.path() + "/MerMI1.cub" };

  UserInterface options(APP_XML, args);



  isisimport(options);



  Pvl output = Pvl(tempDir.path() + "/MerMI1.cub");

  Pvl truth;

  PvlInput >> truth;



  PvlGroup truthDimensions = truth.findGroup("Dimensions", Isis::Plugin::Traverse);

  PvlGroup truthPixels = truth.findGroup("Pixels", Isis::Plugin::Traverse);

  PvlGroup truthArchive = truth.findGroup("Archive", Isis::Plugin::Traverse);

  PvlGroup truthInstrument = truth.findGroup("Instrument", Isis::Plugin::Traverse);

  PvlGroup truthMerImageRequestParms = truth.findGroup("MerImageRequestParms", Isis::Plugin::Traverse);

  PvlGroup truthMerSubframeRequestParms = truth.findGroup("MerSubframeRequestParms", Isis::Plugin::Traverse);

  

  PvlGroup outputDimensions = output.findGroup("Dimensions", Isis::Plugin::Traverse);

  PvlGroup outputPixels = output.findGroup("Pixels", Isis::Plugin::Traverse);

  PvlGroup outputArchive = output.findGroup("Archive", Isis::Plugin::Traverse);

  PvlGroup outputInstrument = output.findGroup("Instrument", Isis::Plugin::Traverse);

  PvlGroup outputMerImageRequestParms = output.findGroup("MerImageRequestParms", Isis::Plugin::Traverse);

  PvlGroup outputMerSubframeRequestParms = output.findGroup("MerSubframeRequestParms", Isis::Plugin::Traverse);



  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputDimensions, truthDimensions);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputPixels, truthPixels);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputArchive, truthArchive);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputInstrument, truthInstrument);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputMerImageRequestParms, truthMerImageRequestParms);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputMerSubframeRequestParms, truthMerSubframeRequestParms);



}



TEST_F(TempTestingFiles, FunctionalTestIsisImportMerMICaseTwo){

    std::istringstream PvlInput(R"(

        Object = IsisCube

          Object = Core

            StartByte   = 65537

            Format      = Tile

            TileSamples = 5

            TileLines   = 5



            Group = Dimensions

              Samples = 5

              Lines   = 5

              Bands   = 1

            End_Group



            Group = Pixels

              Type       = SignedWord

              ByteOrder  = Lsb

              Base       = 0.0

              Multiplier = 1.0

            End_Group

          End_Object



          Group = Instrument

            RoverMotionCounter          = (121, 0, 89, 304, 60)

            RoverMotionCounterName      = (SITE, DRIVE, IDD, PMA, HGA)

            SpacecraftName              = MARS_EXPLORATION_ROVER_2

            InstrumentID                = MI

            InstrumentName              = "MICROSCOPIC IMAGER"

            InstrumentSerialNumber      = 105

            LocalTrueSolarTime          = 11:14:38

            PlanetDayNumber             = 710

            SolarLongitude              = 349.777

            SpacecraftClockCntPartition = 1

            SpacecraftClockStartCount   = 189392700.246

            SpacecraftClockStopCount    = 189392700.886

            StartTime                   = 2006-01-01T13:01:41.933

            StopTime                    = 2006-01-01T13:01:42.573

            ExposureDuration            = 640.0 <ms>

            ExposureDurationCount       = 125

            FilterName                  = MI_OPEN

            FilterNumber                = 2

            FlatFieldCorrectionFlag     = FALSE

            InstrumentModeID            = FULL_FRAME

            InstrumentTemperature       = (-27.0727, -12.3804, -9.42408, -11.0184,

                                          -12.5536, 0.0, -29.7877, -31.0892,

                                          0.0) <degC>

            InstrumentTemperatureName   = ("FRONT HAZ ELECTRONICS",

                                          "REAR HAZ ELECTRONICS",

                                          "LEFT PAN ELECTRONICS", "LEFT PAN CCD",

                                          "RIGHT PAN CCD", "LEFT NAV CCD", "MI CCD",

                                          "MI ELECTRONICS", "EDL CCD")

            OffsetModeID                = 4090

            ShutterEffectCorrectionFlag = TRUE

            TemperatureMiCCD            = -29.7877

            TemperatureMiElectronics    = -31.0892

          End_Group



          Group = Archive

            DataSetID   = MER2-M-MI-2-EDR-SCI-V1.0

            DataSetName = "MER 2 MARS MICROSCOPIC IMAGER SCIENCE EDR VERSION 1.0"

            ProductID   = 2M189392700EFFAL00P2977M2F1

          End_Group



          Group = MerImageRequestParms

            PixelAveragingHeight = 1

            PixelAveragingWidth  = 1

          End_Group



          Group = MerSubframeRequestParms

            FirstLine        = 1

            FirstLineSamples = 1

          End_Group

        End_Object



        Object = Label

          Bytes = 65536

        End_Object



        Object = History

          Name      = IsisCube

          StartByte = 65587

          Bytes     = 542

        End_Object



        Object = OriginalLabel

          Name      = IsisCube

          StartByte = 66129

          Bytes     = 18124

        End_Object

        End

    )");

  QVector<QString> args = {"from=data/isisimport/2M189392700EFFAL00P2977M2F1_cropped.IMG", "to=" + tempDir.path() + "/MerMI2.cub" };

  UserInterface options(APP_XML, args);



  isisimport(options);



  Pvl output = Pvl(tempDir.path() + "/MerMI2.cub");

  Pvl truth;

  PvlInput >> truth;



  PvlGroup truthDimensions = truth.findGroup("Dimensions", Isis::Plugin::Traverse);

  PvlGroup truthPixels = truth.findGroup("Pixels", Isis::Plugin::Traverse);

  PvlGroup truthArchive = truth.findGroup("Archive", Isis::Plugin::Traverse);

  PvlGroup truthInstrument = truth.findGroup("Instrument", Isis::Plugin::Traverse);

  PvlGroup truthMerImageRequestParms = truth.findGroup("MerImageRequestParms", Isis::Plugin::Traverse);

  PvlGroup truthMerSubframeRequestParms = truth.findGroup("MerSubframeRequestParms", Isis::Plugin::Traverse);

  

  PvlGroup outputDimensions = output.findGroup("Dimensions", Isis::Plugin::Traverse);

  PvlGroup outputPixels = output.findGroup("Pixels", Isis::Plugin::Traverse);

  PvlGroup outputArchive = output.findGroup("Archive", Isis::Plugin::Traverse);

  PvlGroup outputInstrument = output.findGroup("Instrument", Isis::Plugin::Traverse);

  PvlGroup outputMerImageRequestParms = output.findGroup("MerImageRequestParms", Isis::Plugin::Traverse);

  PvlGroup outputMerSubframeRequestParms = output.findGroup("MerSubframeRequestParms", Isis::Plugin::Traverse);



  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputDimensions, truthDimensions);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputPixels, truthPixels);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputArchive, truthArchive);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputInstrument, truthInstrument);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputMerImageRequestParms, truthMerImageRequestParms);

  EXPECT_PRED_FORMAT2(AssertPvlGroupEqual, outputMerSubframeRequestParms, truthMerSubframeRequestParms);



}
 
 No newline at end of file

isis/tests/data/isisimport/1M189529263EFF64KCP2977M2F1_cropped.IMG

0 → 100644
+18.1 KiB

File added.

Preview size limit exceeded, changes collapsed.

View file

isis/tests/data/isisimport/2M189392700EFFAL00P2977M2F1_cropped.IMG

0 → 100644
+385 −0

File added.

Preview size limit exceeded, changes collapsed.