Commit 25f373f6 authored by Grégory Mantelet's avatar Grégory Mantelet
Browse files

[TAP] Fix disk space consumption with UPLOAD for synchronous jobs. 

Files uploaded by the user when creating/executing a synchronous job were never
deleted after the job execution.

The same problem applied for the tables already uploaded in the database (in
`TAP_UPLOAD`) when an error occurred before the end of the UPLOAD process.

Now, in case of error when uploading one or more files, or in case of success of
the job, all uploaded files and their corresponding database tables are deleted
after the end of the job.
parent 77257d61

src/tap/TAPSyncJob.java

+57 −33
Original line number Diff line number Diff line
@@ -22,6 +22,7 @@ package tap; 


import java.io.IOException;

import java.util.Date;

import java.util.Iterator;



import javax.servlet.http.HttpServletResponse;
@@ -29,6 +30,7 @@ import tap.parameters.TAPParameters; 
import uws.UWSException;

import uws.job.JobThread;

import uws.service.log.UWSLog.LogLevel;

import uws.service.request.UploadFile;



/**

 * <p>This class represent a TAP synchronous job.
@@ -240,6 +242,9 @@ public class TAPSyncJob { 
		}finally{

			// Whatever the way the execution stops (normal, cancel or error), an execution report must be fulfilled:

			execReport = thread.getExecutionReport();



			// Delete uploaded files:

			deleteUploads(tapParams);

		}



		// Report any error that may have occurred while the thread execution:
@@ -296,6 +301,25 @@ public class TAPSyncJob { 
		return thread.isSuccess();

	}



	/**

	 * Delete all uploaded files.

	 *

	 * @param tapParams	Input parameters (listing all uploaded files, if any).

	 *

	 * @since 2.3

	 */

	protected void deleteUploads(final TAPParameters tapParams){

		Iterator<UploadFile> itFiles = tapParams.getFiles();

		while(itFiles.hasNext()){

			UploadFile uf = itFiles.next();

			try{

				uf.deleteFile();

			}catch(IOException ioe){

				service.getLogger().logTAP(LogLevel.WARNING, this, "END", "Unable to delete the uploaded file \"" + uf.getLocation() + "\"!", ioe);

			}

		}

	}



	/**

	 * <p>Thread which will process the job execution.</p>

	 *

src/tap/upload/Uploader.java

+88 −38
Original line number Diff line number Diff line
@@ -16,13 +16,15 @@ package tap.upload; 
 * You should have received a copy of the GNU Lesser General Public License

 * along with TAPLibrary.  If not, see <http://www.gnu.org/licenses/>.

 *

 * Copyright 2012-2015 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 * Copyright 2012-2018 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 *                       Astronomisches Rechen Institut (ARI)

 */



import java.io.IOException;

import java.io.InputStream;



import com.oreilly.servlet.multipart.ExceededSizeException;



import tap.ServiceConnection;

import tap.ServiceConnection.LimitUnit;

import tap.TAPException;
@@ -31,6 +33,7 @@ import tap.data.LimitedTableIterator; 
import tap.data.TableIterator;

import tap.data.VOTableIterator;

import tap.db.DBConnection;

import tap.db.DBException;

import tap.metadata.TAPColumn;

import tap.metadata.TAPMetadata;

import tap.metadata.TAPMetadata.STDSchema;
@@ -40,18 +43,16 @@ import tap.parameters.DALIUpload; 
import uws.UWSException;

import uws.service.file.UnsupportedURIProtocolException;



import com.oreilly.servlet.multipart.ExceededSizeException;



/**

 * <p>Let create properly given VOTable inputs in the "database".</p>

 * Let create properly given VOTable inputs in the "database".

 *

 * <p>

 * 	This class manages particularly the upload limit in rows and in bytes by creating a {@link LimitedTableIterator}

 * 	with a {@link VOTableIterator}.

 * 	This class manages particularly the upload limit in rows and in bytes by

 * 	creating a {@link LimitedTableIterator} with a {@link VOTableIterator}.

 * </p>

 *

 * @author Gr&eacute;gory Mantelet (CDS;ARI)

 * @version 2.0 (04/2015)

 * @version 2.3 (08/2018)

 *

 * @see LimitedTableIterator

 * @see VOTableIterator
@@ -59,14 +60,17 @@ import com.oreilly.servlet.multipart.ExceededSizeException; 
public class Uploader {

	/** Specification of the TAP service. */

	protected final ServiceConnection service;

	/** Connection to the "database" (which lets upload the content of any given VOTable). */

	/** Connection to the "database" (which lets upload the content of any given

	 * VOTable). */

	protected final DBConnection dbConn;

	/** Description of the TAP_UPLOAD schema to use.

	 * @since 2.0 */

	protected final TAPSchema uploadSchema;

	/** Type of limit to set: ROWS or BYTES. <i>MAY be NULL ; if NULL, no limit will be set.</i> */

	/** Type of limit to set: ROWS or BYTES. <i>MAY be NULL ; if NULL, no limit

	 * will be set.</i> */

	protected final LimitUnit limitUnit;

	/** Limit on the number of rows or bytes (depending of {@link #limitUnit}) allowed to be uploaded in once (whatever is the number of tables). */

	/** Limit on the number of rows or bytes (depending of {@link #limitUnit})

	 * allowed to be uploaded in once (whatever is the number of tables). */

	protected final int limit;



	/** Number of rows already loaded. */
@@ -78,7 +82,8 @@ public class Uploader { 
	 * @param service	Specification of the TAP service using this uploader.

	 * @param dbConn	A valid (open) connection to the "database".

	 *

	 * @throws TAPException	If any error occurs while building this {@link Uploader}.

	 * @throws TAPException	If any error occurs while building this

	 *                     	{@link Uploader}.

	 */

	public Uploader(final ServiceConnection service, final DBConnection dbConn) throws TAPException{

		this(service, dbConn, null);
@@ -90,7 +95,8 @@ public class Uploader { 
	 * @param service	Specification of the TAP service using this uploader.

	 * @param dbConn	A valid (open) connection to the "database".

	 *

	 * @throws TAPException	If any error occurs while building this {@link Uploader}.

	 * @throws TAPException	If any error occurs while building this

	 *                     	{@link Uploader}.

	 *

	 * @since 2.0

	 */
@@ -131,19 +137,30 @@ public class Uploader { 
	}



	/**

	 * <p>Upload all the given VOTable inputs.</p>

	 * Upload all the given VOTable inputs.

	 *

	 * <p><i>Note:

	 * 	The {@link TAPTable} objects representing the uploaded tables will be associated with the TAP_UPLOAD schema specified at the creation of this {@link Uploader}.

	 * 	If no such schema was specified, a default one (whose DB name will be equals to the ADQL name, that's to say {@link STDSchema#UPLOADSCHEMA})

	 * 	is created, will be associated with the uploaded tables and will be returned by this function.

	 * </i></p>

	 * <p><b>Note 1:</b>

	 * 	The {@link TAPTable} objects representing the uploaded tables will be

	 *  associated with the TAP_UPLOAD schema specified at the creation of this

	 *  {@link Uploader}. If no such schema was specified, a default one (whose

	 *  DB name will be equals to the ADQL name, that's to say

	 *  {@link STDSchema#UPLOADSCHEMA}) is created, will be associated with the

	 *  uploaded tables and will be returned by this function.

	 * </p>

	 *

	 * <p><b>Note 2:</b>

	 * 	In case of error while ingesting one or all of the uploaded tables,

	 * 	all tables created in the database before the error occurs are dropped

	 *  <i>(see {@link #dropUploadedTables()})</i>.

	 * </p>

	 *

	 * @param uploads	Array of tables to upload.

	 *

	 * @return	A {@link TAPSchema} containing the list and the description of all uploaded tables.

	 * @return	A {@link TAPSchema} containing the list and the description of

	 *        	all uploaded tables.

	 *

	 * @throws TAPException	If any error occurs while reading the VOTable inputs or while uploading the table into the "database".

	 * @throws TAPException	If any error occurs while reading the VOTable inputs

	 *                     	or while uploading the table into the "database".

	 *

	 * @see DBConnection#addUploadedTable(TAPTable, tap.data.TableIterator)

	 */
@@ -181,14 +198,28 @@ public class Uploader { 
				votable = null;

			}

		}catch(DataReadException dre){

			// Drop uploaded tables:

			dropUploadedTables();

			// Report the error:

			if (dre.getCause() instanceof ExceededSizeException)

				throw dre;

			else

				throw new TAPException("Error while reading the VOTable \"" + tableName + "\": " + dre.getMessage(), dre, UWSException.BAD_REQUEST);

		}catch(IOException ioe){

			// Drop uploaded tables:

			dropUploadedTables();

			// Report the error:

			throw new TAPException("IO error while reading the VOTable of \"" + tableName + "\"!", ioe);

		}catch(UnsupportedURIProtocolException e){

			// Drop uploaded tables:

			dropUploadedTables();

			// Report the error:

			throw new TAPException("URI error while trying to open the VOTable of \"" + tableName + "\"!", e);

		}catch(TAPException te){

			// Drop uploaded tables:

			dropUploadedTables();

			// Report the error:

			throw te;

		}finally{

			try{

				if (dataIt != null)
@@ -200,8 +231,27 @@ public class Uploader { 
			}

		}



		// Return the TAP_UPLOAD schema (containing just the description of the uploaded tables):

		/* Return the TAP_UPLOAD schema (containing just the description of the

		 * uploaded tables): */

		return uploadSchema;

	}



	/**

	 * Drop all tables already uploaded in the database.

	 *

	 * @since 2.3

	 */

	protected void dropUploadedTables(){

		if (uploadSchema == null || uploadSchema.getNbTables() == 0)

			return;



		for(TAPTable table : uploadSchema){

			try{

				dbConn.dropUploadedTable(table);

			}catch(DBException e){

				service.getLogger().error("Unable to drop the uploaded table " + table.getFullName() + "!", e);

			}

		}

	}



}