Commit 0eda76ab authored by Robert Butora's avatar Robert Butora
Browse files

mcutout: fixes too long filename (tar limit is 100bytes), fixes some files not... 

mcutout: fixes too long filename (tar limit is 100bytes), fixes some files not deleted in /srv/cutouts
parent 58bd23d9

data-access/servlet/src/main/java/cutout/webapi/ServletCutout.java

+26 −1
Original line number Diff line number Diff line
@@ -4,6 +4,8 @@ import java.util.logging.Level; 


import java.security.Principal;



import java.time.Instant;



import javax.servlet.ServletException;

import javax.servlet.http.HttpServlet;

import javax.servlet.http.HttpServletRequest;
@@ -159,8 +161,11 @@ public class ServletCutout extends HttpServlet 
            LOGGER.fine("Resolver returns subsurveyId null: no extraCards loaded.");

         }



         String cutAbsPathname = settings.fitsPaths.cutouts() + "/"

                          + generateSubimgPathname(resolver.relPathname(), resolver.hdunum());



         return vlkb.doFile(resolver.relPathname(), resolver.hdunum(),

               pos, band, time, pol, pixels, countNullValues, extraCards);

               pos, band, time, pol, pixels, countNullValues, extraCards, cutAbsPathname);

      }
@@ -337,6 +342,26 @@ public class ServletCutout extends HttpServlet 
   }





   private  String generateSubimgPathname(String relPathname, int hdunum)

   {

      String cutfitsname = "vlkb-cutout";



      Instant instant = Instant.now() ;

      String timestamp = instant.toString().replace(":","-").replace(".","_");



      String tempPathname1 = relPathname.replaceAll("/","-");

      String tempPathname2 = tempPathname1.replaceAll(" ","_");



      if(hdunum == 1)

      {

         return cutfitsname + "_" + timestamp + "_" + tempPathname2;

      }

      else

      {

         String extnum = "EXT" + String.valueOf(hdunum-1);

         return cutfitsname + "_" + timestamp + "_" + extnum + "_" + tempPathname2;

      }

   }

data-access/servlet/src/main/java/mcutout/VlkbCli.java

+308 −322
Original line number Diff line number Diff line
@@ -5,8 +5,6 @@ import java.util.List; 
import java.util.ArrayList;

import java.util.Arrays;



import java.time.Instant;



import java.io.IOException;

import java.io.InputStream;

import java.io.OutputStream;
@@ -22,6 +20,7 @@ import java.nio.file.StandardOpenOption; 
import java.nio.file.Files;

import java.nio.file.Path;

import java.nio.file.Paths;

import org.apache.commons.io.FilenameUtils;



import java.time.*;// Timestamp in cut-filename

import java.io.ByteArrayOutputStream; // for SODA direct streaming doSubimgStream
@@ -80,7 +79,6 @@ class VlkbCli implements Vlkb 
   }







   public CutResult doMerge(String[] idArr, Coord coord, boolean countNullValues)

      throws FileNotFoundException, IOException

   {
@@ -90,110 +88,97 @@ class VlkbCli implements Vlkb 
   }







   public CutResult doFile(String relPathname, int hdunum,

         Pos pos, Band band, Time time, Pol pol, String pixels,

         boolean countNullValues, FitsCard[] extraCards)

         boolean countNullValues, FitsCard[] extraCards,

			String cutAbsPathname)

         throws IOException, InterruptedException

      {

         LOGGER.fine("trace: " + pos.toString() );

         LOGGER.finer("Using doStream() to local file : " + cutAbsPathname);



         CutResult cutResult = new CutResult();



         LOGGER.finer("Using doStream() to local file");



         String absSubimgPathname = settings.fitsPaths.cutouts()

            + "/" + generateSubimgPathname(relPathname, hdunum);



         LOGGER.finer("Uses local filename : " + absSubimgPathname);



         OutputStream fileOutputStream = new FileOutputStream( new File(absSubimgPathname) );



			try(OutputStream fileOutputStream = new FileOutputStream(new File(cutAbsPathname)))

			{

				soda.doStream(relPathname, hdunum, pos, band, time, pol, pixels,fileOutputStream);

			}



         // engine returns absPathname see common/cutout.cpp::do_cutout_file()

         cutResult.fileName = absSubimgPathname;

         cutResult.fileSize = Files.size(Paths.get(absSubimgPathname));

			CutResult cutResult = new CutResult();



			cutResult.fileName = cutAbsPathname;

			cutResult.fileSize = Files.size(Paths.get(cutAbsPathname));

			if(countNullValues)

			{

            cutResult.nullValueCount = doCountNullValues(absSubimgPathname, 1);

				cutResult.nullValueCount = doCountNullValues(cutAbsPathname, 1);

			}



			if(extraCards == null || (extraCards.length < 1))

			{

				LOGGER.finer("Adding extraCards to cut-file implemented only in VlkbAmql");

			}



			cutResult.pixels = null;



			return cutResult;

		}







   private NullValueCount doCountNullValues(String absPathname, int hdunum)

   public MCutResult doMCutout(String jdlJson, String workDir)

		throws IOException, InterruptedException

		{

         ByteArrayOutputStream bos = new ByteArrayOutputStream();

         if(bos == null)

            throw new AssertionError("byte output stream for bounds was not created, is null");

			LOGGER.fine("trace");



         String[] cmdBounds = new String[3];

         cmdBounds[0] = "/usr/local/bin/vlkb";

         cmdBounds[1] = "nullvals";

         cmdBounds[2] = absPathname;

			MCutResult mCutResult;



         ExecCmd exec = new ExecCmd();

         exec.doRun(bos, cmdBounds);

         LOGGER.finest("exec NullVals exitValue: " + exec.exitValue);

			CutArgs[] cutArgsArr = CutArgs.parseCutArgsArr(jdlJson);

			MCutResult.Cut[] cutResultArr = doCutouts( cutArgsArr, workDir );

			String respJsonString = genResponseJson( cutResultArr );

			mCutResult = doCompressCutFiles( cutResultArr, respJsonString );



         bos.close();

			return mCutResult;

		}



         boolean hasResult = (exec.exitValue == 0);

         if(hasResult)





	private MCutResult.Cut[] doCutouts(CutArgs[] cutArgsArr, String workDir)

	{

            String nullValsString = new String(bos.toByteArray());

            LOGGER.finest("vlkb nullvals: " + nullValsString);

		LOGGER.fine("trace, count of cuts : " + String.valueOf( cutArgsArr.length ) );



            if((nullValsString != null) && nullValsString.trim().isEmpty())

		List<MCutResult.Cut> cutResList = new ArrayList<MCutResult.Cut>();



		int ix = 0;

		for(CutArgs cutArgs: cutArgsArr)

		{

               throw new AssertionError("'vlkb nullvals' returned empty string");

            }

			MCutResult.Cut cut = doFileByIdWithErr(cutArgs.id,

					cutArgs.pos, cutArgs.band, cutArgs.time,  cutArgs.pol,  cutArgs.pixels,

					cutArgs.countNullValues, workDir, ix);//, null);//cutArgs.extraCards);



            // parse result: '<fill-ratio> <nullvals-count> <tot-count>'

			cut.index = ix++;



            String[] splitStr = nullValsString.trim().split("\\s+");

            if(splitStr.length != 3)

               throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString);

			LOGGER.finest("cut" + String.valueOf(cut.index) + " : " + cut.content);



            NullValueCount nvc = new NullValueCount();

            nvc.percent = Double.parseDouble(splitStr[0]);

            nvc.nullCount = Long.parseLong(splitStr[1]);

            nvc.totalCount = Long.parseLong(splitStr[2]);

            return nvc;

         }

         else

         {

            throw new AssertionError("'vlkb nullvals' exited without results for: " + absPathname);

         }

			cutResList.add(cut);

		}



		return cutResList.toArray(new MCutResult.Cut[0]);

	}







   private CutResult doFileById(String id, Pos pos, Band band, Time time, Pol pol, String pixels,

         boolean countNullValues/*, Subsurvey[] subsurveys*/)

         throws IOException, InterruptedException

	private MCutResult.Cut doFileByIdWithErr(String id, Pos pos, Band band, Time time, Pol pol,

			String pixels, boolean countNullValues/*, Subsurvey[] subsurveys*/,

			String workDir, int ix)

	{

		LOGGER.fine("trace");



         FitsCard[] extraCards = null;

		MCutResult mCutResult = new MCutResult();

		MCutResult.Cut cut = mCutResult.new Cut(/* FIXME eventually add here new Inputs(id, pos,..) */);



		try

		{

			this.resolver.resolve(id);

			String relPathname = this.resolver.relPathname();

			int hdunum         = this.resolver.hdunum();

			String subsurveyId = this.resolver.obsCollection();



			FitsCard[] extraCards = null;



			if(subsurveyId != null)

			{

				extraCards = Subsurvey.subsurveysFindCards(this.subsurveys, subsurveyId);
@@ -203,78 +188,41 @@ class VlkbCli implements Vlkb 
				LOGGER.finer("Resolver returns subsurveyId null: no extraCards loaded.");

			}



         final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file

			final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from config file



			final int MAX_FILENAME_LEN = 100; // bytes, tar-entry max length is 100bytes

			String cutAbsPathname = workDir + "/"

				+ generateSubimgPathname(ix, relPathname, hdunum, MAX_FILENAME_LEN);



			CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,

               countNullValues, extraCards);

					countNullValues, extraCards, cutAbsPathname);



         return cutResult;

			cut.content     = cutResult.fileName;

			cut.contentType = MCutResult.Cut.ContentType.FILENAME;

		}





   private  String generateSubimgPathname(String relPathname, int hdunum)

   {

      String cutfitsname = "vlkb-cutout";



      Instant instant = Instant.now() ;

      String timestamp = instant.toString().replace(":","-").replace(".","_");



      String tempPathname1 = relPathname.replaceAll("/","-");

      String tempPathname2 = tempPathname1.replaceAll(" ","_");



      if(hdunum == 1)

		catch(MultiValuedParamNotSupported ex)

		{

         return cutfitsname + "_" + timestamp + "_" + tempPathname2;

			cut.content = "MultiValuedParamNotSupported: " + ex.getMessage();

			cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;

			LOGGER.warning(cut.content);

		}

      else

		catch(IllegalArgumentException ex)

		{

         String extnum = "EXT" + String.valueOf(hdunum-1);

         return cutfitsname + "_" + timestamp + "_" + extnum + "_" + tempPathname2;

      }

			cut.content = "IllegalArgumentException: " + ex.getMessage();

			cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;

			LOGGER.warning(cut.content);

		}









   public MCutResult doMCutout(String jdlJson)

         throws IOException, InterruptedException

		catch(Exception ex)

		{

         LOGGER.fine("trace");



         MCutResult mCutResult;



         CutArgs[] cutArgsArr = CutArgs.parseCutArgsArr(jdlJson);

         MCutResult.Cut[] cutResultArr = doCutouts( cutArgsArr );

         String respJsonString = genResponseJson( cutResultArr );

         mCutResult = doCompressCutFiles( cutResultArr, respJsonString );



         return mCutResult;

			cut.content     = "Exception: " + ex.getMessage();

			cut.contentType = MCutResult.Cut.ContentType.SERVICE_ERROR;

			LOGGER.severe(cut.content);

			ex.printStackTrace();

		}







   private MCutResult.Cut[] doCutouts(CutArgs[] cutArgsArr)

   {

      LOGGER.fine("trace, count of cuts : " + String.valueOf( cutArgsArr.length ) );



      List<MCutResult.Cut> cutResList = new ArrayList<MCutResult.Cut>();



      int ix = 0;

      for(CutArgs cutArgs: cutArgsArr)

      {

         MCutResult.Cut cut = doFileByIdWithErr(cutArgs.id,

               cutArgs.pos, cutArgs.band, cutArgs.time,  cutArgs.pol,  cutArgs.pixels,

               cutArgs.countNullValues);//,  null);//cutArgs.extraCards);



         cut.index = ix++;



         LOGGER.finest("cut" + String.valueOf(cut.index) + " : " + cut.content);



         cutResList.add(cut);

		return cut;

	}



      return cutResList.toArray(new MCutResult.Cut[0]);

   }





	// FIXME implement similar for Merge: MCutResult = call-Montage-demosaic-sequence(cutResultArr)
@@ -284,10 +232,12 @@ class VlkbCli implements Vlkb 
			Instant instant = Instant.now();

			String timestamp = new SimpleDateFormat("yyyyMMddHHmmss").format(new Date()) + "_" + instant.getNano();



         final String tgzFileName = settings.fitsPaths.cutouts() + "/mcutout_" + timestamp + ".tar.gz";



			final String tgzFileName = settings.fitsPaths.cutouts()

				+ "/mcutout_" + timestamp + ".tar.gz";

			// generate response-*.json with timestamp

         String respJsonPathname = settings.fitsPaths.cutouts() + "/response_" + timestamp + ".json";

			String respJsonPathname = settings.fitsPaths.cutouts()

				+ "/response_" + timestamp + ".json";



			try (PrintWriter out = new PrintWriter(respJsonPathname))

			{

				out.print(respJsonString);
@@ -299,7 +249,7 @@ class VlkbCli implements Vlkb 


			for(MCutResult.Cut cut : cutArr)

			{

            LOGGER.finest("cut-id"+ String.valueOf(cut.index) + " -> " + cut.content);

				LOGGER.finest("cut-index"+ String.valueOf(cut.index) + " -> " + cut.content);

				if(cut.contentType == MCutResult.Cut.ContentType.FILENAME)

				{

					Path p = Paths.get(cut.content);
@@ -335,49 +285,6 @@ class VlkbCli implements Vlkb 
	}







   private MCutResult.Cut doFileByIdWithErr(String id, Pos pos, Band band, Time time, Pol pol, String pixels,

         boolean countNullValues/*, Subsurvey[] subsurveys*/)

   {

      LOGGER.fine("trace");



      MCutResult mCutResult = new MCutResult();

      MCutResult.Cut cut = mCutResult.new Cut(/* FIXME eventually add here new Inputs(id, pos,..) */);



      try

      {

         CutResult cutResult = doFileById(id,

               pos,  band, time,  pol, pixels,

               countNullValues/*,  subsurveys*/);



         cut.content     = cutResult.fileName;

         cut.contentType = MCutResult.Cut.ContentType.FILENAME;

      }

      catch(MultiValuedParamNotSupported ex) 

      {

         cut.content = "MultiValuedParamNotSupported: " + ex.getMessage();

         cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;

         LOGGER.warning(cut.content);

      }

      catch(IllegalArgumentException ex) 

      {

         cut.content = "IllegalArgumentException: " + ex.getMessage();

         cut.contentType = MCutResult.Cut.ContentType.BAD_REQUEST;

         LOGGER.warning(cut.content);

      }

      catch(Exception ex) 

      {

         cut.content     = "Exception: " + ex.getMessage();

         cut.contentType = MCutResult.Cut.ContentType.SERVICE_ERROR;

         LOGGER.severe(cut.content);

         ex.printStackTrace();

      }



      return cut;

   }







	/*

		<dependency>

		<groupId>org.apache.commons</groupId>
@@ -409,10 +316,7 @@ class VlkbCli implements Vlkb 
							path.getFileName().toString());



					tOut.putArchiveEntry(tarEntry);



               // copy file to TarArchiveOutputStream

               Files.copy(path, tOut);



					Files.copy(path, tOut); // NOTE tar limits max filename length <100bytes

					tOut.closeArchiveEntry();

				}
@@ -420,5 +324,87 @@ class VlkbCli implements Vlkb 
			}

		}





	private NullValueCount doCountNullValues(String absPathname, int hdunum)

		throws IOException, InterruptedException

		{

			ByteArrayOutputStream bos = new ByteArrayOutputStream();

			if(bos == null)

				throw new AssertionError("byte output stream for bounds was not created, is null");



			String[] cmdBounds = new String[3];

			cmdBounds[0] = "/usr/local/bin/vlkb";

			cmdBounds[1] = "nullvals";

			cmdBounds[2] = absPathname;



			ExecCmd exec = new ExecCmd();

			exec.doRun(bos, cmdBounds);

			LOGGER.finest("exec NullVals exitValue: " + exec.exitValue);



			bos.close();



			boolean hasResult = (exec.exitValue == 0);

			if(hasResult)

			{

				String nullValsString = new String(bos.toByteArray());

				LOGGER.finest("vlkb nullvals: " + nullValsString);



				if((nullValsString != null) && nullValsString.trim().isEmpty())

				{

					throw new AssertionError("'vlkb nullvals' returned empty string");

				}



				// parse result: '<fill-ratio> <nullvals-count> <tot-count>'



				String[] splitStr = nullValsString.trim().split("\\s+");

				if(splitStr.length != 3)

					throw new AssertionError("'vlkb nullvals' did not return 3 numbers but: " + nullValsString);



				NullValueCount nvc = new NullValueCount();

				nvc.percent = Double.parseDouble(splitStr[0]);

				nvc.nullCount = Long.parseLong(splitStr[1]);

				nvc.totalCount = Long.parseLong(splitStr[2]);

				return nvc;

			}

			else

			{

				throw new AssertionError("'vlkb nullvals' exited without results for: " + absPathname);

			}

		}







	private  String generateSubimgPathname(int ix, String relPathname, int hdunum, int maxLen)

	{

		String cutfitsname = String.valueOf(ix);



		String tempPathname1 = relPathname.replaceAll("/","-");

		String tempPathname2 = tempPathname1.replaceAll(" ","_");

		String filename;



		if(hdunum == 1)

		{

			filename = cutfitsname + "_" + tempPathname2;

		}

		else

		{

			String extnum = "E" + String.valueOf(hdunum-1);

			filename = cutfitsname + "_" + extnum + "_" + tempPathname2 ;

		}



		// shorten filename to maxLen



		if(filename.length() > maxLen) // FIXME consider UTF-8 ? string-len != string-buff-len

		{

			String ext  = FilenameUtils.getExtension(filename);// without dot

			String name = FilenameUtils.removeExtension(filename);



			filename = name.substring(0, maxLen - 1 - ext.length() ) + "." + ext;

		}



		return filename;

	}





}

data-access/servlet/src/main/java/mcutout/webapi/UWSMCutoutWork.java

+23 −17
Original line number Diff line number Diff line
@@ -10,6 +10,8 @@ import java.io.OutputStreamWriter; 
import java.io.BufferedReader;

import java.io.InputStreamReader;



import org.apache.commons.io.FileUtils;



import uws.UWSException;

import uws.job.ErrorType;

import uws.job.JobThread;
@@ -78,7 +80,10 @@ public class UWSMCutoutWork extends JobThread 
         }

         String reqJsonString = jsonStringBuffer.toString();



         MCutResult mresult = vlkb.doMCutout(reqJsonString);

			String workDir = settings.fitsPaths.cutouts() + "/" + job.getJobId();

			FileUtils.forceMkdir(new File(workDir));



         MCutResult mresult = vlkb.doMCutout(reqJsonString, workDir);



			logger.logThread(LogLevel.INFO, this, "Prepare Result",

					" job:requestUrl: " +  job.getUrl().getRequestURL(), null);
@@ -88,9 +93,11 @@ public class UWSMCutoutWork extends JobThread 


			// delete cut-files (were published/copied to uws-file store) 



			mresult.deleteResJson();

			mresult.deleteCutFiles();

			//mresult.deleteResJson();

			//mresult.deleteCutFiles();



			FileUtils.deleteDirectory(new File(workDir));

			//FileUtils.deleteQuietly(new File(workDir));// FIXME Never throws except. leaves the dir there if error



			/* FIXME here was uws-check is-job-Interrupted */

		}
@@ -109,13 +116,12 @@ public class UWSMCutoutWork extends JobThread 
			result.setMimeType(mimeType);



			File downloadFile = new File(absPathname);

		FileInputStream fileInput = new FileInputStream(downloadFile);



		OutputStream resOutputStream = getResultOutput(result);

			try(FileInputStream fileInput = new FileInputStream(downloadFile);

					OutputStream resOutputStream = getResultOutput(result))

			{

				fileInput.transferTo(resOutputStream);



				publishResult(result);



			}

			downloadFile.delete();

		}

data-access/servlet/src/main/java/vlkb/Vlkb.java

+3 −2
Original line number Diff line number Diff line
@@ -15,12 +15,13 @@ public interface Vlkb 
{

   public CutResult doFile(String relPathname, int hdunum,

         Pos pos, Band band, Time time, Pol pol, String pixels,

         boolean countNullValues, FitsCard[] extraCards)

         boolean countNullValues, FitsCard[] extraCards,

			String cutAbsPathname)

         throws IOException, InterruptedException;







   public MCutResult doMCutout(String jdlJson)

   public MCutResult doMCutout(String jdlJson, String workDir)

      throws IOException, InterruptedException;

data-access/servlet/src/main/java/vlkb/VlkbAmqp.java

+3 −3
Original line number Diff line number Diff line
@@ -79,7 +79,7 @@ class VlkbAmqp implements Vlkb 


   public CutResult doFile(String relPathname, int hdunum,

      Pos pos, Band band, Time time, Pol pol, String pixels,

      boolean countNullValues, FitsCard[] extraCards)

      boolean countNullValues, FitsCard[] extraCards, String dummyCutAbsPathname)

         throws IOException, InterruptedException

      {

         LOGGER.fine("trace: " + pos.toString() );
@@ -134,7 +134,7 @@ class VlkbAmqp implements Vlkb 
         final String DEFAULT_TIME_SYSTEM = "MJD_UTC"; // FIXME take from confif file



         CutResult cutResult = doFile(relPathname, hdunum, pos, band, time, pol, pixels,

               countNullValues, extraCards);

               countNullValues, extraCards, null);



         return cutResult;

      }
@@ -145,7 +145,7 @@ class VlkbAmqp implements Vlkb 
   ///////////////////////////////////////////////////////////////////////////////////





   public MCutResult doMCutout(String jdlJson)

   public MCutResult doMCutout(String jdlJson, String workDir)

         throws IOException

      {

         LOGGER.fine("trace");