moves doFile doMCutout from CutoutImpl -> DatasetsCli DatasetsAmqp (30e19f82) · Commits · ViaLactea / vlkb-soda

data-access/servlet/src/main/java/cutout/Cutout.java

+0 −11

Original line number	Diff line number	Diff line
		@@ -18,16 +18,5 @@ public interface Cutout
		Pos pos, Band band, Time time, Pol pol, String pixels,
		OutputStream outputStream) throws IOException, InterruptedException;


		public CutResult doFile(String relPathname, int hdunum,
		Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, FitsCard[] extraCards)
		throws IOException, InterruptedException;


		public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		throws IOException, InterruptedException;

		}

data-access/servlet/src/main/java/cutout/CutoutImpl.java

+0 −190

Original line number	Diff line number	Diff line
		@@ -276,195 +276,5 @@ class CutoutImpl implements Cutout
		return other.split("\\s+");
		}


		private NullValueCount doCountNullValues(String absPathname, int hdunum)
		throws IOException, InterruptedException
		{
		ByteArrayOutputStream bos = new ByteArrayOutputStream();
		if(bos == null)
		throw new AssertionError("byte output stream for bounds was not created, is null");

		String[] cmdBounds = new String[3];
		cmdBounds[0] = "/usr/local/bin/vlkb";
		cmdBounds[1] = "nullvals";
		cmdBounds[2] = absPathname;

		ExecCmd exec = new ExecCmd();
		exec.doRun(bos, cmdBounds);
		LOGGER.info("exec NullVals exitValue: " + exec.exitValue);

		bos.close();

		boolean hasResult = (exec.exitValue == 0);
		if(hasResult)
		{
		String nullValsString = new String(bos.toByteArray());
		LOGGER.info("vlkb nullvals: " + nullValsString);

		if((nullValsString != null) && nullValsString.trim().isEmpty())
		{
		throw new AssertionError("'vlkb nullvals' returned empty string");
		}

		// parse result: '<fill-ratio> <nullvals-count> <tot-count>'

		String[] splitStr = nullValsString.trim().split("\\s+");
		if(splitStr.length != 3) throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString);

		NullValueCount nvc = new NullValueCount();
		nvc.percent = Double.parseDouble(splitStr[0]);
		nvc.nullCount = Long.parseLong(splitStr[1]);
		nvc.totalCount = Long.parseLong(splitStr[2]);
		return nvc;
		}
		else
		{
		throw new AssertionError("'vlkb nullvals' exited without results for: " + absPathname);
		}
		}



		public CutResult doFile(String relPathname, int hdunum,
		Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, FitsCard[] extraCards)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace: " + pos.toString() );

		CutResult cutResult = new CutResult();

		if(settings.amqpConn.isHostnameEmpty())
		{
		LOGGER.info("Using doStream() to local file");

		String absSubimgPathname = settings.fitsPaths.cutouts()
		+ "/" + generateSubimgPathname(relPathname, hdunum);

		LOGGER.info("Uses local filename : " + absSubimgPathname);

		OutputStream fileOutputStream = new FileOutputStream( new File(absSubimgPathname) );

		doStream(relPathname, hdunum, pos, band, time, pol, pixels, fileOutputStream);

		// engine returns absPathname see common/cutout.cpp::do_cutout_file()
		cutResult.fileName = absSubimgPathname;
		cutResult.fileSize = Files.size(Paths.get(absSubimgPathname));

		if(countNullValues)
		{
		cutResult.nullValueCount = doCountNullValues(absSubimgPathname, 1);
		}

		if(extraCards == null \|\| (extraCards.length < 1))
		{
		LOGGER.info("Adding extraCards to cut-file not implemented when using 'vlkb' exec (implemented in engine vlkbd/AMQP)");
		}

		cutResult.pixels = null;
		}
		else
		{
		LOGGER.info("Using AMQP");

		JsonEncoder jReq = new JsonEncoder();
		jReq.add(relPathname, hdunum);
		jReq.add(pos);
		jReq.add(band);
		jReq.add(time);
		jReq.add(pol);

		// jReq.add(pixels), FIXME implement to supoort PIXLES in vlkb-legacy by AMQP

		jReq.add(countNullValues);
		jReq.add(extraCards);

		String outJson = RpcOverAmqp.doRpc( settings.amqpConn, jReq.toString() );

		cutResult = JsonDecoder.responseFromCutoutJson( outJson );
		}

		return cutResult;
		}



		public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace");

		String relPathname;
		int hdunum;

		FitsCard[] extraCards = null;

		String dbUri = settings.dbConn.uri();

		if(settings.dbConn.isDbUriEmpty())
		{
		Resolver rsl = new ResolverFromId();
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();

		/* FIXME needs also match on filename - some subsurveys have the same storage-path,
		* and file-filter distiguishes frequences
		* OR
		* ivoid must include obs-collection

		Path path = Paths.get(rsl.relPathname());
		String storagePath = path.getParent().toString();
		extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
		*/
		}
		else
		{
		ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();
		String subsurveyId = rsl.obsCollection();
		if(subsurveyId != null)
		{
		extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
		}
		else
		{
		LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
		}
		}

		final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file

		CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,
		countNullValues, extraCards);

		return cutResult;
		}


		private String generateSubimgPathname(String relPathname, int hdunum)
		{
		String cutfitsname = "vlkb-cutout";

		Instant instant = Instant.now() ;
		String timestamp = instant.toString().replace(":","-").replace(".","_");

		String tempPathname1 = relPathname.replaceAll("/","-");
		String tempPathname2 = tempPathname1.replaceAll(" ","_");

		if(hdunum == 1)
		{
		return cutfitsname + "_" + timestamp + "_" + tempPathname2;
		}
		else
		{
		String extnum = "EXT" + String.valueOf(hdunum-1);
		return cutfitsname + "_" + timestamp + "_" + extnum + "_" + tempPathname2;
		}
		}

		}

data-access/servlet/src/main/java/cutout/Datasets.java

+15 −2

Original line number	Diff line number	Diff line
		@@ -9,15 +9,28 @@ import java.io.InputStreamReader;
		import java.io.PrintWriter;
		import java.time.Instant;//Timestamp in cut-filename

		import vo.parameter.*;

		public interface Datasets
		{
		public CutResult doFile(String relPathname, int hdunum,
		Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, FitsCard[] extraCards)
		throws IOException, InterruptedException;


		public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		throws IOException, InterruptedException;

		public CutResult doMerge(String[] idArr, Coord coord, boolean countNullValues)
		throws FileNotFoundException, IOException;


		public MCutResult doMCutout(String jdlJson)
		throws IOException, InterruptedException;



		public CutResult doMerge(String[] idArr, Coord coord, boolean countNullValues)
		throws FileNotFoundException, IOException;
		}

data-access/servlet/src/main/java/cutout/DatasetsAmqp.java

+94 −0

Original line number	Diff line number	Diff line
		@@ -67,6 +67,100 @@ class DatasetsAmqp implements Datasets
		}


		///////////////////////////////////////////////////////////////////////////////////

		public CutResult doFile(String relPathname, int hdunum,
		Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, FitsCard[] extraCards)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace: " + pos.toString() );

		CutResult cutResult = new CutResult();

		LOGGER.info("Using AMQP");

		JsonEncoder jReq = new JsonEncoder();
		jReq.add(relPathname, hdunum);
		jReq.add(pos);
		jReq.add(band);
		jReq.add(time);
		jReq.add(pol);

		// jReq.add(pixels), FIXME implement to supoort PIXLES in vlkb-legacy by AMQP

		jReq.add(countNullValues);
		jReq.add(extraCards);

		String outJson = RpcOverAmqp.doRpc( settings.amqpConn, jReq.toString() );

		cutResult = JsonDecoder.responseFromCutoutJson( outJson );

		return cutResult;
		}



		public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace");

		String relPathname;
		int hdunum;

		FitsCard[] extraCards = null;

		String dbUri = settings.dbConn.uri();

		if(settings.dbConn.isDbUriEmpty())
		{
		Resolver rsl = new ResolverFromId();
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();

		/* FIXME needs also match on filename - some subsurveys have the same storage-path,
		* and file-filter distiguishes frequences
		* OR
		* ivoid must include obs-collection

		Path path = Paths.get(rsl.relPathname());
		String storagePath = path.getParent().toString();
		extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
		*/
		}
		else
		{
		ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();
		String subsurveyId = rsl.obsCollection();
		if(subsurveyId != null)
		{
		extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
		}
		else
		{
		LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
		}
		}

		final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file

		CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,
		countNullValues, extraCards);

		return cutResult;
		}




		///////////////////////////////////////////////////////////////////////////////////


		public MCutResult doMCutout(String jdlJson)
		throws IOException

data-access/servlet/src/main/java/cutout/DatasetsCli.java

+177 −5

Original line number	Diff line number	Diff line
		@@ -16,6 +16,7 @@ import java.io.PrintWriter;
		import java.io.BufferedReader;
		import java.io.File;
		import java.io.FileInputStream;
		import java.io.FileOutputStream;
		import java.io.FileNotFoundException;
		import java.nio.file.StandardOpenOption;
		import java.nio.file.Files;
		@@ -81,6 +82,177 @@ class DatasetsCli implements Datasets



		///////////////////////////////////////////////////////////////////////////////////////


		private NullValueCount doCountNullValues(String absPathname, int hdunum)
		throws IOException, InterruptedException
		{
		ByteArrayOutputStream bos = new ByteArrayOutputStream();
		if(bos == null)
		throw new AssertionError("byte output stream for bounds was not created, is null");

		String[] cmdBounds = new String[3];
		cmdBounds[0] = "/usr/local/bin/vlkb";
		cmdBounds[1] = "nullvals";
		cmdBounds[2] = absPathname;

		ExecCmd exec = new ExecCmd();
		exec.doRun(bos, cmdBounds);
		LOGGER.info("exec NullVals exitValue: " + exec.exitValue);

		bos.close();

		boolean hasResult = (exec.exitValue == 0);
		if(hasResult)
		{
		String nullValsString = new String(bos.toByteArray());
		LOGGER.info("vlkb nullvals: " + nullValsString);

		if((nullValsString != null) && nullValsString.trim().isEmpty())
		{
		throw new AssertionError("'vlkb nullvals' returned empty string");
		}

		// parse result: '<fill-ratio> <nullvals-count> <tot-count>'

		String[] splitStr = nullValsString.trim().split("\\s+");
		if(splitStr.length != 3) throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString);

		NullValueCount nvc = new NullValueCount();
		nvc.percent = Double.parseDouble(splitStr[0]);
		nvc.nullCount = Long.parseLong(splitStr[1]);
		nvc.totalCount = Long.parseLong(splitStr[2]);
		return nvc;
		}
		else
		{
		throw new AssertionError("'vlkb nullvals' exited without results for: " + absPathname);
		}
		}


		public CutResult doFile(String relPathname, int hdunum,
		Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, FitsCard[] extraCards)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace: " + pos.toString() );

		CutResult cutResult = new CutResult();

		LOGGER.info("Using doStream() to local file");

		String absSubimgPathname = settings.fitsPaths.cutouts()
		+ "/" + generateSubimgPathname(relPathname, hdunum);

		LOGGER.info("Uses local filename : " + absSubimgPathname);

		OutputStream fileOutputStream = new FileOutputStream( new File(absSubimgPathname) );

		cutout.doStream(relPathname, hdunum, pos, band, time, pol, pixels, fileOutputStream);

		// engine returns absPathname see common/cutout.cpp::do_cutout_file()
		cutResult.fileName = absSubimgPathname;
		cutResult.fileSize = Files.size(Paths.get(absSubimgPathname));

		if(countNullValues)
		{
		cutResult.nullValueCount = doCountNullValues(absSubimgPathname, 1);
		}

		if(extraCards == null \|\| (extraCards.length < 1))
		{
		LOGGER.info("Adding extraCards to cut-file not implemented when using 'vlkb' exec (implemented in engine vlkbd/AMQP)");
		}

		cutResult.pixels = null;
		return cutResult;
		}


		public CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,
		boolean countNullValues, Subsurvey[] subsurveys)
		throws IOException, InterruptedException
		{
		LOGGER.info("trace");

		String relPathname;
		int hdunum;

		FitsCard[] extraCards = null;

		String dbUri = settings.dbConn.uri();

		if(settings.dbConn.isDbUriEmpty())
		{
		Resolver rsl = new ResolverFromId();
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();

		/* FIXME needs also match on filename - some subsurveys have the same storage-path,
		* and file-filter distiguishes frequences
		* OR
		* ivoid must include obs-collection

		Path path = Paths.get(rsl.relPathname());
		String storagePath = path.getParent().toString();
		extraCards = Subsurvey.subsurveysFindCardsByStoragePath(subsurveys, storagePath);
		*/
		}
		else
		{
		ResolverByObsCore rsl = new ResolverByObsCore(settings.dbConn, subsurveys);
		rsl.resolve(id);
		relPathname = rsl.relPathname();
		hdunum = rsl.hdunum();
		String subsurveyId = rsl.obsCollection();
		if(subsurveyId != null)
		{
		extraCards = Subsurvey.subsurveysFindCards(subsurveys, subsurveyId);
		}
		else
		{
		LOGGER.info("Resolver with Obscore returns subsurveyId null: no extraCards loaded.");
		}
		}

		final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file

		CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,
		countNullValues, extraCards);

		return cutResult;
		}

		private String generateSubimgPathname(String relPathname, int hdunum)
		{
		String cutfitsname = "vlkb-cutout";

		Instant instant = Instant.now() ;
		String timestamp = instant.toString().replace(":","-").replace(".","_");

		String tempPathname1 = relPathname.replaceAll("/","-");
		String tempPathname2 = tempPathname1.replaceAll(" ","_");

		if(hdunum == 1)
		{
		return cutfitsname + "_" + timestamp + "_" + tempPathname2;
		}
		else
		{
		String extnum = "EXT" + String.valueOf(hdunum-1);
		return cutfitsname + "_" + timestamp + "_" + extnum + "_" + tempPathname2;
		}
		}


		///////////////////////////////////////////////////////////////////////////////////////




		public MCutResult doMCutout(String jdlJson)
		throws IOException, InterruptedException
		{
		@@ -190,7 +362,7 @@ class DatasetsCli implements Datasets

		try
		{
		CutResult cutResult = cutout.doFileById(id,
		CutResult cutResult = doFileById(id,
		pos, band, time, pol, pixels,
		countNullValues, subsurveys);