Update ISIS Serial numbers notebook to run

%% Cell type:code id: tags:

%% Cell type:code id: tags:

``` python

``` python

import os

import os

import sys

import sys

print(os.path.abspath('../'))

print(os.path.abspath('../'))

sys.path.insert(0, os.path.abspath('../'))

sys.path.insert(0, os.path.abspath('../'))

from functools import reduce

from functools import reduce

import glob

import glob

import re

import re

from autocnet.graph import network

from autocnet.graph import network

from autocnet.fileio.sqlalchemy_json.alchemy import NestedJsonObject

from autocnet.fileio.sqlalchemy_json.alchemy import NestedJsonObject

from autocnet.utils.utils import find_in_dict

from autocnet.utils.utils import find_in_dict

import sqlalchemy

import sqlalchemy

from sqlalchemy.ext import declarative

from sqlalchemy.ext import declarative

from sqlalchemy import orm

from sqlalchemy import orm

import json

import json

import warnings

import warnings

import pvl

import pvl

from pvl._collections import PVLModule

from pvl._collections import PVLModule

```

```

%% Output

%% Output

    /Users/jlaura/github/autocnet

    /Users/jlaura/github/autocnet

%% Cell type:markdown id: tags:

%% Cell type:markdown id: tags:

## Generate the data

## Generate the data

The next three cells take a directory of cleaned `.trn` files (extra values like AUTO or OPTIONAL removed) and generate a `data.db` database of ISIS serial number translations.

The next three cells take a directory of cleaned `.trn` files (extra values like AUTO or OPTIONAL removed) and generate a `data.db` database of ISIS serial number translations.

%% Cell type:code id: tags:

%% Cell type:code id: tags:

``` python

``` python

# Database population and a declared class

# Database population and a declared class

Base = declarative.declarative_base()

Base = declarative.declarative_base()

class Translations(Base):

class Translations(Base):

    __tablename__ = 'isis_translations'

    __tablename__ = 'isis_translations'

    id = sqlalchemy.Column(sqlalchemy.INTEGER, primary_key=True)

    id = sqlalchemy.Column(sqlalchemy.INTEGER, primary_key=True)

    mission = sqlalchemy.Column(sqlalchemy.String)

    mission = sqlalchemy.Column(sqlalchemy.String)

    instrument = sqlalchemy.Column(sqlalchemy.String)

    instrument = sqlalchemy.Column(sqlalchemy.String)

    translation = sqlalchemy.Column(NestedJsonObject)

    translation = sqlalchemy.Column(NestedJsonObject)

    def __init__(self, mission, instrument, translation):

    def __init__(self, mission, instrument, translation):

        self.mission = mission

        self.mission = mission

        self.instrument = instrument

        self.instrument = instrument

        self.translation = translation

        self.translation = translation

class StringToMission(Base):

class StringToMission(Base):

    __tablename__ = 'isis_mission_to_standard'

    __tablename__ = 'isis_mission_to_standard'

    id = sqlalchemy.Column(sqlalchemy.INTEGER, primary_key=True)

    id = sqlalchemy.Column(sqlalchemy.INTEGER, primary_key=True)

    key = sqlalchemy.Column(sqlalchemy.String)

    key = sqlalchemy.Column(sqlalchemy.String)

    value = sqlalchemy.Column(sqlalchemy.String)

    value = sqlalchemy.Column(sqlalchemy.String)

    def __init__(self, key, value):

    def __init__(self, key, value):

        self.key = key

        self.key = key

        self.value = value

        self.value = value

engine = sqlalchemy.create_engine('sqlite:///data.db')

engine = sqlalchemy.create_engine('sqlite:///data.db')

Base.metadata.bind = engine

Base.metadata.bind = engine

Base.metadata.create_all()

Base.metadata.create_all()

session = orm.sessionmaker(bind=engine)()

session = orm.sessionmaker(bind=engine)()

```

```

%% Cell type:code id: tags:

%% Cell type:code id: tags:

``` python

``` python

files = glob.glob('../autocnet/examples/serial_number_translations/*.trn')

files = glob.glob('../autocnet/examples/serial_number_translations/*.trn')

for f in files:

for f in files:

    p = pvl.load(f)

    p = pvl.load(f)

    name = os.path.basename(f[:-16])

    name = os.path.basename(f[:-16])

    try:

    try:

        v = re.findall("([a-z, 0-9]*)([A-Z, a-z, 0-9]*)", name)[0]

        v = re.findall("([a-z, 0-9]*)([A-Z, a-z, 0-9]*)", name)[0]

        mission, instrument = v

        mission, instrument = v

    except:

    except:

        v = re.findall("[a-z, 0-9]*", name)

        v = re.findall("[a-z, 0-9]*", name)

        mission = v[0]

        mission = v[0]

        instrument = None

        instrument = None

    r = Translations(mission, instrument, p)

    r = Translations(mission, instrument, p)

    session.add(r)

    session.add(r)

session.commit()

session.commit()

```

```

%% Cell type:code id: tags:

%% Cell type:code id: tags:

``` python

``` python

# Build the mission names lookup table

# Build the mission names lookup table

v = """

v = """

Group = MissionName

Group = MissionName

  InputKey      = SpacecraftName

  InputKey      = SpacecraftName

  InputGroup    = "IsisCube,Instrument"

  InputGroup    = "IsisCube,Instrument"

  InputPosition = (IsisCube, Instrument)

  InputPosition = (IsisCube, Instrument)

  Translation   = (Aircraft, "Aircraft")

  Translation   = (Aircraft, "Aircraft")

  Translation   = (Apollo15, "APOLLO 15")

  Translation   = (Apollo15, "APOLLO 15")

  Translation   = (Apollo15, "APOLLO15")

  Translation   = (Apollo15, "APOLLO15")

  Translation   = (Apollo16, "APOLLO 16")

  Translation   = (Apollo16, "APOLLO 16")

  Translation   = (Apollo16, "APOLLO16")

  Translation   = (Apollo16, "APOLLO16")

  Translation   = (Apollo17, "APOLLO 17")

  Translation   = (Apollo17, "APOLLO 17")

  Translation   = (Apollo17, "APOLLO17")

  Translation   = (Apollo17, "APOLLO17")

  Translation   = (Cassini, Cassini-Huygens)

  Translation   = (Cassini, Cassini-Huygens)

#  Translation   = (Chan1, "CHANDRAYAAN-1 ORBITER")

#  Translation   = (Chan1, "CHANDRAYAAN-1 ORBITER")

#  Translation   = (Chan1, CHANDRAYAAN1_ORBITER)

#  Translation   = (Chan1, CHANDRAYAAN1_ORBITER)

#  Translation   = (Chan1, CHANDRAYAAN-1)

#  Translation   = (Chan1, CHANDRAYAAN-1)

  Translation   = (Chandrayaan1, "CHANDRAYAAN-1 ORBITER")

  Translation   = (Chandrayaan1, "CHANDRAYAAN-1 ORBITER")

  Translation   = (Chandrayaan1, CHANDRAYAAN1_ORBITER)

  Translation   = (Chandrayaan1, CHANDRAYAAN1_ORBITER)

  Translation   = (Chandrayaan1, CHANDRAYAAN-1)

  Translation   = (Chandrayaan1, CHANDRAYAAN-1)

  Translation   = (Clementine1, CLEMENTINE_1)

  Translation   = (Clementine1, CLEMENTINE_1)

  Translation   = (Clementine1, "CLEMENTINE 1")

  Translation   = (Clementine1, "CLEMENTINE 1")

  Translation   = (Dawn, "DAWN")

  Translation   = (Dawn, "DAWN")

  Translation   = (Galileo, "Galileo Orbiter")

  Translation   = (Galileo, "Galileo Orbiter")

  Translation   = (Hayabusa, HAYABUSA)

  Translation   = (Hayabusa, HAYABUSA)

  Translation   = (Ideal, IdealSpacecraft)

  Translation   = (Ideal, IdealSpacecraft)

  Translation   = (Kaguya, KAGUYA)

  Translation   = (Kaguya, KAGUYA)

  Translation   = (Kaguya, SELENE-M)

  Translation   = (Kaguya, SELENE-M)

  Translation   = (Lo, "Lunar Orbiter 3")

  Translation   = (Lo, "Lunar Orbiter 3")

  Translation   = (Lo, "Lunar Orbiter 4")

  Translation   = (Lo, "Lunar Orbiter 4")

  Translation   = (Lo, "Lunar Orbiter 5")

  Translation   = (Lo, "Lunar Orbiter 5")

  Translation   = (Lro, "LUNAR RECONNAISSANCE ORBITER")

  Translation   = (Lro, "LUNAR RECONNAISSANCE ORBITER")

  Translation   = (Lro, "Lunar Reconnaissance Orbiter")

  Translation   = (Lro, "Lunar Reconnaissance Orbiter")

  Translation   = (Mariner10, Mariner_10)

  Translation   = (Mariner10, Mariner_10)

  Translation   = (Mariner10, MARINER_10)

  Translation   = (Mariner10, MARINER_10)

  Translation   = (Mer, "MARS EXPLORATION ROVER 1")

  Translation   = (Mer, "MARS EXPLORATION ROVER 1")

  Translation   = (Mer, MARS_EXPLORATION_ROVER_1)

  Translation   = (Mer, MARS_EXPLORATION_ROVER_1)

  Translation   = (Mer, "MARS EXPLORATION ROVER 2")

  Translation   = (Mer, "MARS EXPLORATION ROVER 2")

  Translation   = (Mer, "SIMULATED MARS EXPLORATION ROVER 1")

  Translation   = (Mer, "SIMULATED MARS EXPLORATION ROVER 1")

  Translation   = (Mer, "SIMULATED MARS EXPLORATION ROVER 2")

  Translation   = (Mer, "SIMULATED MARS EXPLORATION ROVER 2")

  Translation   = (Messenger, MESSENGER)

  Translation   = (Messenger, MESSENGER)

  Translation   = (Messenger, Messenger)

  Translation   = (Messenger, Messenger)

  Translation   = (Mex, "MARS EXPRESS")

  Translation   = (Mex, "MARS EXPRESS")

  Translation   = (Mex, "Mars Express")

  Translation   = (Mex, "Mars Express")

    Translation   = (Mgs, MARSGLOBALSURVEYOR)

    Translation   = (Mgs, MARSGLOBALSURVEYOR)

  Translation   = (Mgs, "MARS GLOBAL SURVEYOR")

  Translation   = (Mgs, "MARS GLOBAL SURVEYOR")

  Translation   = (Mro, "MARS RECONNAISSANCE ORBITER")

  Translation   = (Mro, "MARS RECONNAISSANCE ORBITER")

  Translation   = (Mro, Mars_Reconnaissance_Orbiter)

  Translation   = (Mro, Mars_Reconnaissance_Orbiter)

  Translation   = (NewHorizons, "NEW HORIZONS")

  Translation   = (NewHorizons, "NEW HORIZONS")

  Translation   = (Near, NEAR)

  Translation   = (Near, NEAR)

  Translation   = (Near, "NEAR EARTH ASTEROID RENDEZVOUS")

  Translation   = (Near, "NEAR EARTH ASTEROID RENDEZVOUS")

  Translation   = (Odyssey, MARS_ODYSSEY)

  Translation   = (Odyssey, MARS_ODYSSEY)

  Translation   = (OsirisRex, OSIRIS-REX)

  Translation   = (OsirisRex, OSIRIS-REX)

  Translation   = (Smart1, SMART1)

  Translation   = (Smart1, SMART1)

  Translation   = (Viking1, VIKING_ORBITER_1)

  Translation   = (Viking1, VIKING_ORBITER_1)

  Translation   = (Viking2, VIKING_ORBITER_2)

  Translation   = (Viking2, VIKING_ORBITER_2)

  Translation   = (Voyager1, VOYAGER_1)

  Translation   = (Voyager1, VOYAGER_1)

  Translation   = (Voyager2, VOYAGER_2)

  Translation   = (Voyager2, VOYAGER_2)

End_Group

End_Group

End

End

"""

"""

p = pvl.loads(v)

p = pvl.loads(v)

for k, v in p['MissionName'].items():

for k, v in p['MissionName'].items():

    if k == 'Translation':

    if k == 'Translation':

        r = StringToMission(v[1], v[0])

        r = StringToMission(v[1], v[0])

        session.add(r)

        session.add(r)

session.commit()

session.commit()

```

```

%% Cell type:markdown id: tags:

%% Cell type:markdown id: tags:

## Sample Query

## Sample Query

%% Cell type:code id: tags:

%% Cell type:code id: tags:

``` python

``` python

# Sample querying the database

# Sample querying the database

for i, j, t in session.query(Translations.mission, Translations.instrument, Translations.translation):

for i, j, t in session.query(Translations.mission, Translations.instrument, Translations.translation):

    print(i,j)

    print(i,j)

    d = PVLModule(t)

    d = PVLModule(t)

    print(d)

    print(d)

    break

    break

```

```

%% Cell type:markdown id: tags:

%% Cell type:markdown id: tags:

## Testing code to prototype the functionality now in autocnet

## Testing code to prototype the functionality now in autocnet

%% Cell type:code id: tags:

%% Cell type:code id: tags:

``` python

``` python

 class SerialNumberDecoder(pvl.decoder.PVLDecoder):

 class SerialNumberDecoder(pvl.decoder.PVLDecoder):

     """

     """

     A PVL Decoder class to handle cube label parsing for the purpose of creating a valid ISIS

     A PVL Decoder class to handle cube label parsing for the purpose of creating a valid ISIS

     serial number. Inherits from the PVLDecoder in planetarypy's pvl module.

     serial number. Inherits from the PVLDecoder in planetarypy's pvl module.

     """

     """

     def cast_unquoated_string(self, value):

     def cast_unquoated_string(self, value):

         """

         """

         Overrides the parent class's method so that any un-quoted string type value found in the

         Overrides the parent class's method so that any un-quoted string type value found in the

         parsed pvl will just return the original value. This is needed so that keyword values

         parsed pvl will just return the original value. This is needed so that keyword values

         are not re-formatted from what is originally in the ISIS cube label.

         are not re-formatted from what is originally in the ISIS cube label.

         Note: This affects value types that are recognized as null, boolean, number, datetime,

         Note: This affects value types that are recognized as null, boolean, number, datetime,

         et at.

         et at.

         """

         """

         return value.decode('utf-8')

         return value.decode('utf-8')

def get_isis_translation(label):

def get_isis_translation(label):

    """

    """

    Compute the ISIS serial number for a given image using

    Compute the ISIS serial number for a given image using

    the input cube or the label extracted from the cube.

    the input cube or the label extracted from the cube.

    Parameters

    Parameters

    ----------

    ----------

    label : dict or str

    label : dict or str

            A PVL dict object or file name to extract

            A PVL dict object or file name to extract

            the PVL object from

            the PVL object from

    Returns

    Returns

    -------

    -------

    translation : dict

    translation : dict

                  A PVLModule object containing the extracted

                  A PVLModule object containing the extracted

                  translation file

                  translation file

    """

    """

    if not isinstance(label, PVLModule):

    if not isinstance(label, PVLModule):

        label = pvl.load(label)

        label = pvl.load(label)

    cube_obj = find_in_dict(label, 'Instrument')

    cube_obj = find_in_dict(label, 'Instrument')

    # Grab the spacecraft name and run it through the ISIS lookup

    # Grab the spacecraft name and run it through the ISIS lookup

    spacecraft_name = find_in_dict(cube_obj, 'SpacecraftName')

    spacecraft_name = find_in_dict(cube_obj, 'SpacecraftName')

    for row in session.query(StringToMission).filter(StringToMission.key==spacecraft_name):

    for row in session.query(StringToMission).filter(StringToMission.key==spacecraft_name):

        spacecraft_name = row.value.lower()

        spacecraft_name = row.value.lower()

    #Try and pull an instrument identifier

    #Try and pull an instrument identifier

    try:

    try:

        instrumentid = find_in_dict(cube_obj, 'InstrumentId').capitalize()

        instrumentid = find_in_dict(cube_obj, 'InstrumentId').capitalize()

    except:

    except:

        instrumentid = None

        instrumentid = None

    # Grab the translation PVL object using the lookup

    # Grab the translation PVL object using the lookup

    for row in session.query(Translations).filter(Translations.mission==spacecraft_name,

    for row in session.query(Translations).filter(Translations.mission==spacecraft_name,

                                                  Translations.instrument==instrumentid):

                                                  Translations.instrument==instrumentid):

        # Convert the JSON back to a PVL object

        # Convert the JSON back to a PVL object

        translation = PVLModule(row.translation)

        translation = PVLModule(row.translation)

    return translation

    return translation

def extract_subgroup(data, key_list):

def extract_subgroup(data, key_list):

    return reduce(lambda d, k: d[k], key_list, data)

    return reduce(lambda d, k: d[k], key_list, data)

def generate_serial_number(label):

def generate_serial_number(label):

    if not isinstance(label, PVLModule):

    if not isinstance(label, PVLModule):

        label = pvl.load(label, cls=SerialNumberDecoder)

        label = pvl.load(label, cls=SerialNumberDecoder)

    # Get the translation information

    # Get the translation information

    translation = get_isis_translation(label)

    translation = get_isis_translation(label)

    serial_number = []

    serial_number = []

    # Sort the keys to ensure proper iteration order

    # Sort the keys to ensure proper iteration order

    keys = sorted(translation.keys())

    keys = sorted(translation.keys())

    for k in keys:

    for k in keys:

        group = translation[k]

        group = translation[k]

        search_key = group['InputKey']

        search_key = group['InputKey']

        search_position = group['InputPosition']

        search_position = group['InputPosition']

        search_translation = {group['Translation'][1]:group['Translation'][0]}

        search_translation = {group['Translation'][1]:group['Translation'][0]}

        print(search_key, search_position, search_translation)

        print(search_key, search_position, search_translation)

        sub_group = extract_subgroup(label, search_position)

        sub_group = extract_subgroup(label, search_position)

        serial_entry = sub_group[search_key]

        serial_entry = sub_group[search_key]

        if serial_entry in search_translation.keys():

        if serial_entry in search_translation.keys():

            serial_entry = search_translation[serial_entry]

            serial_entry = search_translation[serial_entry]

        elif '*' in search_translation.keys() and search_translation['*'] != '*':

        elif '*' in search_translation.keys() and search_translation['*'] != '*':

            serial_entry = search_translation['*']

            serial_entry = search_translation['*']

        serial_number.append(serial_entry)

        serial_number.append(serial_entry)

    return '/'.join(serial_number)

    return '/'.join(serial_number)

serial = generate_serial_number('/Users/jlaura/Desktop/Apollo15/AS15-M-0296_sub4.cub')

print(serial)

```

```

%% Output

%% Output

    SpacecraftName ['IsisCube', 'Instrument'] {'*': 'APOLLO15'}

    SpacecraftName ['IsisCube', 'Instrument'] {'*': 'APOLLO15'}

    InstrumentId ['IsisCube', 'Instrument'] {'*': '*'}

    InstrumentId ['IsisCube', 'Instrument'] {'*': '*'}

    StartTime ['IsisCube', 'Instrument'] {'*': '*'}

    StartTime ['IsisCube', 'Instrument'] {'*': '*'}

    APOLLO15/METRIC/1971-07-31T01:24:36.970

    APOLLO15/METRIC/1971-07-31T01:24:36.970

%% Cell type:markdown id: tags:

Edit below code for the correct path to an Apollo cube and uncomment to test. Must remain commented out for a non-failing build server-side.

%% Cell type:code id: tags:

``` python

#serial = generate_serial_number('/Users/jlaura/Desktop/Apollo15/AS15-M-0296_sub4.cub')

#print(serial)

```