Adding data ingestion script and resources

import xml.etree.ElementTree as ET

import glob

from datetime import datetime

from imagedb.models import *

flist = glob.glob('resources/*.xml')

try:

  inst = Instrument.objects.get(instrumentName="NISP")

except:

  inst = Instrument.objects.create(telescopeName='Euclid', instrumentName='NISP')

for f in flist:

  root = ET.parse(f)

  exposureTime = float(root.find(".//ExposureTime/Value").text)

  print(root.find(".//OBT").text) 

  observationDate = datetime.strptime(root.find(".//OBT").text, 

                                      "%Y-%m-%dT%H:%M:%S.%f")

  obs_id = int(root.find(".//FieldId").text)

  dither = int(root.find(".//DitherObservation").text)

  ra = float(root.find(".//TargetPointing/RA").text)

  dec = float(root.find(".//TargetPointing/Dec").text)

  orientation = float(root.find(".//TargetPointing/Orientation").text)

  filterWheelPos = root.find(".//Filter").text[4]

  file_name = root.find(".//DataStorage/DataContainer/FileName").text

  file_name = file_name.replace("NIR_W-CALIB","LE1_NISP")

  data = DataContainer(

    fileFormat = 'fits',

    formatIdentifier = 'le1.nisprawframe',

    formatVersion = '1.0',

    url = "http://ia2-owncloud.oats.inaf.it/fake/7ff2f203/data/" + file_name    

  )

  data.save()

  image = NispRawFrame(

    exposureTime = exposureTime,

    imgNumber = 16,

    naxis1 = 2040,

    naxis2 = 2040,

    imageType = {'category':'SCIENCE', 

                 'firstType':'OBJECT', 

                 'secondType':'SKY'},

    observationDateTime = observationDate,

    observationId = obs_id,

    ditherNumber = dither,

    instrument = inst,

    commandedPointing = {'rightAscension':ra,

                         'declination':dec,

                         'orientation':orientation},

    filterWheelPosition = filterWheelPos,

    grismWheelPosition = 'OPEN'   

  )

  image.frameFile = data

  image.save()

  detList = root.findall('.//Detector')

  detList = sorted(detList, key = lambda x : x.find(".//DetectorId").text)

  for d in detList:

    det_id = d.find(".//DetectorId").text

    gain = float(d.find(".//Gain").text)

    readoutNoise = float(d.find(".//ReadoutNoise").text)

    detector = image.detectors.create(

      detectorId = det_id,

      gain = gain,

      readoutNoise = readoutNoise

    )

    coordinate1 = d.find(".//CTYPE1/CoordinateType").text

    projection1 = d.find(".//CTYPE1/ProjectionType").text

    coordinate2 = d.find(".//CTYPE2/CoordinateType").text

    projection2 = d.find(".//CTYPE2/ProjectionType").text

    crval1 = float(d.find(".//CRVAL1").text)

    crval2 = float(d.find(".//CRVAL2").text)

    crpix1 = float(d.find(".//CRPIX1").text)

    crpix2 = float(d.find(".//CRPIX2").text)

    cd1_1 = float(d.find(".//CD1_1").text)

    cd1_2 = float(d.find(".//CD1_2").text)

    cd2_1 = float(d.find(".//CD2_1").text)

    cd2_2 = float(d.find(".//CD2_2").text)

    Astrometry.objects.create(

      ctype1 = {'coordinateType': coordinate1, 'projectionType': 'TAN'},

      ctype2 = {'coordinateType': coordinate2, 'projectionType': 'TAN'},

      crval1 = crval1,

      crval2 = crval2,

      crpix1 = crpix1,

      crpix2 = crpix2,

      cd1_1 = cd1_1,

      cd1_2 = cd1_2,

      cd2_1 = cd2_1,

      cd2_2 = cd2_2,

      detector = detector

    )