mcutout: implements doCutouts loop with Cutout::doFile and CutArgs JSON parsing (632db6a9) · Commits · ViaLactea / vlkb-soda

data-access/servlet/src/main/java/cutout/CutArgs.java

0 → 100644

+165 −0

Original line number	Diff line number	Diff line

		import java.util.logging.Logger;


		import java.util.Iterator;
		import java.util.List;
		import java.util.ArrayList;
		/* 'JSON-Simple' library */
		import org.json.simple.JSONArray;
		import org.json.simple.JSONObject;
		import org.json.simple.parser.JSONParser;
		import org.json.simple.parser.ParseException;

		import vo.parameter.*;

		// The cutout-args:
		// CutResult doCutoutFile(
		// String id,
		// Pos pos, Band band, Time time, Pol pol, String pixels,
		// boolean countNullValues)
		//
		// The result:
		//class CutResult
		//{
		// String fileName;
		// long fileSize;
		// NullValueCount nullValueCount;
		// String pixels; <- from WCS-conversion (not input)
		//}
		//
		//class NullValueCount
		//{
		// double percent;
		// long nullCount;
		// long totalCount;
		//}
		//


		// vlkb-soda has:
		//class Coord
		//{
		// Pos pos;
		// Band band;
		// Time time;
		// Pol pol;
		// String pixels;
		//}




		// FIXME or clla this Jdl.java ?
		class CutArgs
		{
		static final Logger LOGGER = Logger.getLogger("CutArgs");

		static CutArgs[] parseCutArgsArr(String jsonString)
		{
		LOGGER.info("trace : " + jsonString);

		List<CutArgs> argsList = new ArrayList<CutArgs>();

		JSONParser parser = new JSONParser();
		try
		{
		// FIXME explicit converions: should check type first
		JSONArray a = (JSONArray) parser.parse(jsonString);

		for (Object o : a)
		{
		// FIXME check type
		JSONObject jElem = (JSONObject) o;
		LOGGER.info("jElem: " + jElem.toString());

		CutArgs args = new CutArgs();

		args.id = (String) jElem.get("id");
		if(args.id == null)
		args.id = (String) jElem.get("pubdid");

		Object jCNV = jElem.get("countNullVals");
		args.countNullValues = (jCNV == null) ? false : (Boolean) jElem.get("countNullVals");

		// FIXME first check if array (or null?) and then cast

		String defaultPosSystem = "GALACTIC";
		Pos pos = Pos.parsePosCirclePolygonFromJson(jElem.toString(), defaultPosSystem);
		args.pos = pos;

		String defaultBandSystem = "VELO_LSRK";
		Band band = Band.parseBandFromJson(jElem.toString(), defaultBandSystem);
		args.band = band;

		// TBD: * legacy is parsed inside POS and BAND, not here -> do modif
		// * still missing TIME and POL and ID and (?)countNullValues


		// FIXME explicit converions: check if exists and then check type first
		JSONObject jCoord = (JSONObject) jElem.get("coord");

		if(jCoord != null)
		{
		// legacy
		LOGGER.info("TEST: " + jCoord.toString());

		args.lon = (Double) jCoord.get("l");
		args.lat = (Double) jCoord.get("b");
		args.radius = (Double) jCoord.get("r");
		args.dl = (Double) jCoord.get("dl");
		args.db = (Double) jCoord.get("db");
		args.vl = (Double) jCoord.get("vl");
		args.vu = (Double) jCoord.get("vu");
		}

		// FIXME missing:
		String skySystem = "GALACTIC";
		String specSystem = "VELO_LSRK";


		// convert:

		if(args.lon != null && args.lat != null && args.radius != null)
		{
		Circle circle = new Circle(args.lon, args.lat, args.radius);
		args.pos = new Pos(circle, skySystem);
		}

		if(args.lon != null && args.lat != null && args.dl != null && args.db != null)
		{
		//Range range(args.lon - args.dl/2.0, args.lon + args.dl/2.0,
		// args.lat - args.db/2.0, args.lat + args.db/2.0);,
		Range range = new Range(args.lon, args.lat, args.dl, args.db);
		args.pos = new Pos(range, skySystem);
		}

		if(args.vl != null && args.vu != null)
		{
		args.band = new Band(args.vl, args.vu, specSystem);
		}

		argsList.add(args);
		}
		}
		catch(ParseException e)
		{
		e.printStackTrace();
		}

		return argsList.toArray(new CutArgs[0]);
		}

		String id;
		Pos pos;
		Band band;
		Time time;
		Pol pol;
		String pixels;
		// FitxCard[] extraCards; ???
		Boolean countNullValues;

		// legacy
		Double lon, lat, radius, dl, db, vl, vu; // double throws except if json-null
		}

data-access/servlet/src/main/java/cutout/Cutout.java

+4 −0

Original line number	Diff line number	Diff line
		@@ -25,5 +25,9 @@ public interface Cutout
		throws IOException, InterruptedException;


		public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		throws IOException, InterruptedException;

		}

data-access/servlet/src/main/java/cutout/CutoutImpl.java

+67 −1

Original line number	Diff line number	Diff line
		@@ -33,13 +33,14 @@ class CutoutImpl implements Cutout
		static final Logger LOGGER = Logger.getLogger(DatasetsImpl.class.getName());

		private Settings settings = null;
		// private Subsurvey[] subsurveys = null;
		private Subsurvey[] subsurveys = null;


		public CutoutImpl()
		{
		LOGGER.info("trace DatasetsImpl()");
		this.settings = Settings.getInstance();
		this.subsurveys = null;
		}


		@@ -47,6 +48,14 @@ class CutoutImpl implements Cutout
		{
		LOGGER.info("trace DatasetsImpl(settings)");
		this.settings = settings;
		this.subsurveys = null;
		}

		public CutoutImpl(Settings settings, Subsurvey[] subsurveys)
		{
		LOGGER.info("trace DatasetsImpl(settings)");
		this.settings = settings;
		this.subsurveys = subsurveys;
		}


		@@ -380,6 +389,63 @@ class CutoutImpl implements Cutout



		public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace");

		String relPathname;
		int hdunum;

		FitsCard[] extraCards = null;

		String dbUri = settings.dbConn.uri();

		if(settings.dbConn.isDbUriEmpty())
		{
		Resolver rsl = new ResolverFromId();
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();

		/* FIXME needs also match on filename - some subsurveys have the same storage-path,
		* and file-filter distiguishes frequences
		* OR
		* ivoid must include obs-collection

		Path path = Paths.get(rsl.relPathname());
		String storagePath = path.getParent().toString();
		extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
		*/
		}
		else
		{
		ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();
		String subsurveyId = rsl.obsCollection();
		if(subsurveyId != null)
		{
		extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
		}
		else
		{
		LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
		}
		}

		final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file

		CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,
		countNullValues, extraCards);

		return cutResult;
		}



		private String doRpc(String InStr)
		{
		final String userName = "guest";

data-access/servlet/src/main/java/cutout/DatasetsImpl.java

+100 −39

Original line number	Diff line number	Diff line
		@@ -25,18 +25,21 @@ import java.nio.file.Paths;
		import java.time.*;// Timestamp in cut-filename
		import java.io.ByteArrayOutputStream; // for SODA direct streaming doSubimgStream

		import vo.parameter.*;

		class DatasetsImpl implements Datasets
		{
		static final Logger LOGGER = Logger.getLogger(DatasetsImpl.class.getName());

		private Settings settings = null;
		private Subsurvey[] subsurveys = null;

		private Cutout cutout = null;

		public DatasetsImpl()
		{
		LOGGER.info("trace DatasetsImpl()");
		this.settings = Settings.getInstance();
		cutout = new CutoutImpl(settings, subsurveys);
		}


		@@ -44,6 +47,7 @@ class DatasetsImpl implements Datasets
		{
		LOGGER.info("trace DatasetsImpl(settings)");
		this.settings = settings;
		this.cutout = new CutoutImpl(settings, subsurveys);
		}


		@@ -52,6 +56,7 @@ class DatasetsImpl implements Datasets
		LOGGER.info("trace DatasetsImpl(settings, subsurveys)");
		this.settings = settings;
		this.subsurveys = subsurveys;
		this.cutout = new CutoutImpl(settings, subsurveys);
		}


		@@ -72,22 +77,25 @@ class DatasetsImpl implements Datasets
		{
		LOGGER.info("trace");

		String updatedJsonString = JdlMCutout.resolveAndUpdateJsonRequest(jdlJson, settings, subsurveys);

		String outJson;
		MCutResult mCutResult;

		if(settings.amqpConn.isHostnameEmpty())
		{
		LOGGER.info("doMCutout with CLI");
		outJson = doCli( JdlMCutout.mcutoutToJson(updatedJsonString) );
		CutArgs[] cutArgsArr = CutArgs.parseCutArgsArr(jdlJson);
		MCutResult.Cut[] cutResultArr = doCutouts( cutArgsArr );
		mCutResult = doCompressCutFiles( cutResultArr );
		}
		else
		{
		LOGGER.info("doMCutout over AMQP");
		outJson = doRpc( JdlMCutout.mcutoutToJson(updatedJsonString) );
		String updatedJsonString = JdlMCutout.resolveAndUpdateJsonRequest(jdlJson, settings, subsurveys);
		LOGGER.info("doMCutout over AMQP : " + updatedJsonString);
		String outJson = doRpc( JdlMCutout.mcutoutToJson(updatedJsonString) );
		mCutResult = JdlMCutout.responseFromMCutoutJson(outJson);
		}

		return JdlMCutout.responseFromMCutoutJson(outJson);
		return mCutResult;
		}


		@@ -95,9 +103,94 @@ class DatasetsImpl implements Datasets
		/* ================= ALL ================================== */


		private MCutResult.Cut[] doCutouts(CutArgs[] cutArgsArr)
		{
		LOGGER.info("trace, count of cuts : " + String.valueOf( cutArgsArr.length ) );

		List<MCutResult.Cut> cutResList = new ArrayList<MCutResult.Cut>();

		int ix = 0;
		for(CutArgs cutArgs: cutArgsArr)
		{
		MCutResult.Cut cut = doFileByIdWithErr(cutArgs.id,
		cutArgs.pos, cutArgs.band, cutArgs.time, cutArgs.pol, cutArgs.pixels,
		cutArgs.countNullValues, null);//cutArgs.extraCards);

		cut.index = ix++;

		LOGGER.info("cut" + String.valueOf(cut.index) + " : " + cut.content);

		cutResList.add(cut);
		}

		return cutResList.toArray(new MCutResult.Cut[0]);
		}


		// FIXME implement similar for Merge: MCutResult = call-Montage-demosaic-sequence(cutResultArr)
		private MCutResult doCompressCutFiles(MCutResult.Cut[] cutArr)
		{
		// FIXME do compression here
		for(MCutResult.Cut cut : cutArr)
		{
		LOGGER.info("TBD compress cut-id"+ String.valueOf(cut.index) + " -> " + cut.content);
		}

		MCutResult mCutResult = new MCutResult();
		mCutResult.cutResArr = cutArr;
		mCutResult.fileName = "filename.tar.gz"; // FIXME do-zip-all-cuts(cutResultArr)
		mCutResult.fileSize = 0;

		return mCutResult;
		}


		private MCutResult.Cut doFileByIdWithErr(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		{
		LOGGER.info("trace");

		MCutResult mCutResult = new MCutResult();
		MCutResult.Cut cut = mCutResult.new Cut(/* FIXME eventually add here new Inputs(id, pos,..) */);

		try
		{
		CutResult cutResult = cutout.doFileById(id,
		pos, band, time, pol, pixels,
		countNullValues, subsurveys);

		cut.content = cutResult.fileName;
		cut.contentType = MCutResult.Cut.ContentType.FILENAME;
		}
		catch(MultiValuedParamNotSupported ex)
		{
		cut.content = "MultiValuedParamNotSupported: " + ex.getMessage();
		cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;
		LOGGER.info(cut.content);
		}
		catch(IllegalArgumentException ex)
		{
		cut.content = "IllegalArgumentException: " + ex.getMessage();
		cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;
		LOGGER.info(cut.content);
		}
		catch(Exception ex)
		{
		cut.content = "Exception: " + ex.getMessage();
		cut.contentType = MCutResult.Cut.ContentType.SERVICE_ERROR;
		LOGGER.info(cut.content);
		ex.printStackTrace();
		}

		return cut;
		}



		private String doRpc(String InStr)
		{
		LOGGER.info("trace");

		final String userName = "guest";
		final String password = "guest";
		// FIXME move these to Settings
		@@ -139,38 +232,6 @@ class DatasetsImpl implements Datasets



		private String doCli(String inStr)
		{

		LOGGER.info("doMCutout in CLI/ExecCmd not implemented");

		/*
		List<CutResult> cutResList = new ArrayList<CutResult>();

		Coord[] coords = Jdl.parseReq(inStr);

		for(Coord coord: coords)
		{
		CutResult cutResult = doFile(coord.relPathname, coord.hdunum,
		coord.pos, coord.band, coord.time, coord.pol, coord.pixels,
		coord.countNullValues, coord.extraCards);

		cutResList.add(cutResult);
		}

		cutResultArr = cutResList.toArray()

		MCutResult = do-zip-all-cuts(cutResultArr)
		// similar in Merge: CutResult = call-Montage-demosaic-sequence(cutResultArr)

		String outStr = Jdl.encodeToJson(MCutResult);
		*/

		return null;//outStr;
		}



		private String generateSubimgPathname(String relPathname, int hdunum)
		{
		String cutfitsname = "vlkb-cutout";

data-access/servlet/src/main/java/webapi/ServletCutout.java

+3 −50

Original line number	Diff line number	Diff line
		@@ -57,10 +57,10 @@ public class ServletCutout extends javax.servlet.http.HttpServlet
		boolean showDuration = settings.defaults.showDuration;
		long startTime_msec;

		protected Cutout cutout = new CutoutImpl(settings);

		private Subsurvey[] subsurveys = null;

		protected Cutout cutout = new CutoutImpl(settings, subsurveys);

		public void init() throws ServletException
		{
		super.init();
		@@ -129,60 +129,13 @@ public class ServletCutout extends javax.servlet.http.HttpServlet
		}



		protected CutResult doCutoutFile(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace");

		String relPathname;
		int hdunum;

		FitsCard[] extraCards = null;

		String dbUri = settings.dbConn.uri();

		if(settings.dbConn.isDbUriEmpty())
		{
		Resolver rsl = new ResolverFromId();
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();

		/* FIXME needs also match on filename - some subsurveys have the same storage-path,
		* and file-filter distiguishes frequences
		* OR
		* ivoid must include obs-collection

		Path path = Paths.get(rsl.relPathname());
		String storagePath = path.getParent().toString();
		extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
		*/
		}
		else
		{
		ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();
		String subsurveyId = rsl.obsCollection();
		if(subsurveyId != null)
		{
		extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
		}
		else
		{
		LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
		}
		}

		final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file

		CutResult cutResult = cutout.doFile(relPathname, hdunum, pos, band, time, pol, pixels,
		countNullValues, extraCards);

		return cutResult;
		return cutout.doFileById(id, pos, band, time, pol, pixels, countNullValues, subsurveys);
		}