Commit 7bd91a1c authored by Grégory Mantelet's avatar Grégory Mantelet
Browse files

[UWS,TAP] Fix the configuration file for the UPLOAD feature. 

The property `upload_default_db_limit` has been deprecated. Indeed, in the
current state of the TAP protocol, this makes no sense: the user can not change
the limit size (in bytes or rows) for uploaded tables.

The property `upload_max_file_size` has been deprecated. It is actually
duplicated: `upload_max_db_limit`, if expressed in bytes already lets put a
limit on the maximum size of an uploaded table/file.

The property `upload_max_request_size` has been added. It lets set a maximum
size for a whole HTTP Multipart Request. By default it is set to 250MB.

The default value of `upload_max_db_size` is now 1 million rows.

The UPLOAD feature is still disabled by default (i.e. `upload_enabled=false`).
parent 02a4a7f5

src/tap/ServiceConnection.java

+209 −156
Original line number Original line Diff line number Diff line
@@ -16,13 +16,14 @@ package tap; 
 * You should have received a copy of the GNU Lesser General Public License

 * You should have received a copy of the GNU Lesser General Public License

 * along with TAPLibrary.  If not, see <http://www.gnu.org/licenses/>.

 * along with TAPLibrary.  If not, see <http://www.gnu.org/licenses/>.

 *

 *

 * Copyright 2012-2015 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 * Copyright 2012-2018 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 *                       Astronomisches Rechen Institut (ARI)

 *                       Astronomisches Rechen Institut (ARI)

 */

 */





import java.util.Collection;

import java.util.Collection;

import java.util.Iterator;

import java.util.Iterator;





import adql.db.FunctionDef;

import tap.db.DBConnection;

import tap.db.DBConnection;

import tap.formatter.OutputFormat;

import tap.formatter.OutputFormat;

import tap.log.DefaultTAPLog;

import tap.log.DefaultTAPLog;
@@ -31,19 +32,19 @@ import tap.metadata.TAPMetadata; 
import uws.service.UserIdentifier;

import uws.service.UserIdentifier;

import uws.service.file.LocalUWSFileManager;

import uws.service.file.LocalUWSFileManager;

import uws.service.file.UWSFileManager;

import uws.service.file.UWSFileManager;

import adql.db.FunctionDef;





/**

/**

 * <p>Description and parameters list of a TAP service.</p>

 * Description and parameters list of a TAP service.

 *

 *

 * <p>

 * <p>

 * 	Through this object, it is possible to configure the different limits and formats,

 * 	Through this object, it is possible to configure the different limits and

 * 	but also to list all available tables and columns, to declare geometry features as all allowed user defined functions

 * 	formats, but also to list all available tables and columns, to declare

 * 	and to say where log and other kinds of files must be stored.

 * 	geometry features as well as all allowed user defined functions and to say

 * 	where log and other kinds of files must be stored.

 * </p>

 * </p>

 *

 *

 * @author Gr&eacute;gory Mantelet (CDS;ARI)

 * @author Gr&eacute;gory Mantelet (CDS;ARI)

 * @version 2.0 (03/2015)

 * @version 2.3 (09/2018)

 */

 */

public interface ServiceConnection {

public interface ServiceConnection {
@@ -103,14 +104,46 @@ public interface ServiceConnection { 




		/**

		/**

		 * Compares the 2 given values (each one expressed in the given unit).

		 * Compares the 2 given values (each one expressed in the given unit).

		 * Conversions are done internally in order to make a correct comparison between the 2 limits.

		 *

		 * <p>

		 * 	Conversions are done internally in order to make a correct

		 * 	comparison between the 2 limits.

		 * </p>

		 *

		 *

		 * @param leftLimit		Value/Limit of the comparison left part.

		 * @param leftLimit		Value/Limit of the comparison left part.

		 * @param leftUnit		Unit of the comparison left part value.

		 * @param leftUnit		Unit of the comparison left part value.

		 * @param rightLimit	Value/Limit of the comparison right part.

		 * @param rightLimit	Value/Limit of the comparison right part.

		 * @param rightUnit		Unit of the comparison right part value.

		 * @param rightUnit		Unit of the comparison right part value.

		 *

		 *

		 * @return the value 0 if x == y; a value less than 0 if x < y; and a value greater than 0 if x > y

		 * @return	the value 0 if x == y; a value less than 0 if x < y;

		 *        	and a value greater than 0 if x > y

		 *

		 * @throws TAPException If the two given units are not compatible.

		 *

		 * @see #compare(long, LimitUnit, long, LimitUnit)

		 *

		 * @since 1.1

		 */

		public static int compare(final int leftLimit, final LimitUnit leftUnit, final int rightLimit, final LimitUnit rightUnit) throws TAPException{

			return compare(leftLimit, leftUnit, rightLimit, rightUnit);

		}



		/**

		 * Compares the 2 given values (each one expressed in the given unit).

		 *

		 * <p>

		 * 	Conversions are done internally in order to make a correct

		 * 	comparison between the 2 limits.

		 * </p>

		 *

		 * @param leftLimit		Value/Limit of the comparison left part.

		 * @param leftUnit		Unit of the comparison left part value.

		 * @param rightLimit	Value/Limit of the comparison right part.

		 * @param rightUnit		Unit of the comparison right part value.

		 *

		 * @return	the value 0 if x == y;

		 *        	a value less than 0 if x < y;

		 *        	and a value greater than 0 if x > y

		 *

		 *

		 * @throws TAPException If the two given units are not compatible.

		 * @throws TAPException If the two given units are not compatible.

		 *

		 *
@@ -119,9 +152,9 @@ public interface ServiceConnection { 
		 * @see Integer#compare(int, int)

		 * @see Integer#compare(int, int)

		 * @see Long#compare(long, long)

		 * @see Long#compare(long, long)

		 *

		 *

		 * @since 1.1

		 * @since 2.3

		 */

		 */

		public static int compare(final int leftLimit, final LimitUnit leftUnit, final int rightLimit, final LimitUnit rightUnit) throws TAPException{

		public static int compare(final long leftLimit, final LimitUnit leftUnit, final long rightLimit, final LimitUnit rightUnit) throws TAPException{

			if (!leftUnit.isCompatibleWith(rightUnit))

			if (!leftUnit.isCompatibleWith(rightUnit))

				throw new TAPException("Limit units (" + leftUnit + " and " + rightUnit + ") are not compatible!");

				throw new TAPException("Limit units (" + leftUnit + " and " + rightUnit + ") are not compatible!");
@@ -365,13 +398,16 @@ public interface ServiceConnection { 




	/**

	/**

	 * <i><b>[MANDATORY]</b></i>

	 * <i><b>[MANDATORY]</b></i>

	 * <p>This function let enable or disable the upload capability of this TAP service.</p>

	 * <p>This function lets enable or disable the upload capability of this TAP

	 * service.</p>

	 *

	 *

	 * <p><i>Note:

	 * <p><i>Note:

	 * 	If the upload is disabled, the request is aborted and an HTTP-400 error is thrown each time some tables are uploaded.

	 * 	If the upload is disabled, the request is aborted and an HTTP-400 error

	 * 	is thrown each time some tables are uploaded.

	 * </i></p>

	 * </i></p>

	 *

	 *

	 * @return	<i>true</i> to enable the upload capability, <i>false</i> to disable it.

	 * @return	<i>true</i> to enable the upload capability,

	 *        	<i>false</i> to disable it.

	 */

	 */

	public boolean uploadEnabled();

	public boolean uploadEnabled();
@@ -389,47 +425,67 @@ public interface ServiceConnection { 
	 * 	<li><b>Exactly 2 values or a NULL object is expected here.</b></li>

	 * 	<li><b>Exactly 2 values or a NULL object is expected here.</b></li>

	 * 	<li><b>If NULL</b>, the upload limit is not limited and uploads could be

	 * 	<li><b>If NULL</b>, the upload limit is not limited and uploads could be

	 * 	    theoretically unlimited.</li>

	 * 	    theoretically unlimited.</li>

	 * 	<li><b>If not NULL</b>, the 2 values must correspond to the default upload limit

	 * 	<li><b>If not NULL</b>, the 2 values must correspond to the default

	 * 	    and the maximum upload limit.</li>

	 * 	    upload limit and the maximum upload limit.</li>

	 * 	<li><b>The default value</b> is used inform the user about the server wishes.</li>

	 * 	<li><b>The default value</b> is used to inform the user about the server

	 * 	<li><b>The maximum value</b> is used to really limit the upload limit.</li>

	 * 	    wishes. <i><b>In this version of TAPLib, this value is never used,

	 * 	<li><b>The structure of the object</b> returned by this function MUST be the same as the object returned by {@link #getUploadLimitType()}.

	 * 		whatever is its value.</b> It is defined here only in case the

	 * 	    Particularly, the type given by the N-th item of {@link #getUploadLimitType()} must correspond to the N-th limit returned by this function.</li>

	 * 		TAP protocol evolves and allow a such default value for UPLOAD

	 * 	<li><b>The default value</b> MUST be less or equals the maximum value.</li>

	 * 		limits.</i></li>

	 * 	<li><b>Both values must be positive</b>. If a negative value is given it will be interpreted as "no limit".</li>

	 * 	<li><b>The maximum value</b> is used to really limit the upload

	 * 	    limit.</li>

	 * 	<li><b>The structure of the object</b> returned by this function MUST be

	 * 	    the same as the object returned by {@link #getUploadLimitType()}.

	 * 	    Particularly, the type given by the N-th item of

	 * 	    {@link #getUploadLimitType()} must correspond to the N-th limit

	 * 	    returned by this function.</li>

	 * 	<li><b>The default value</b> MUST be less or equals the maximum

	 * 	    value.</li>

	 * 	<li><b>Both values must be positive</b>. If a negative value is given it

	 * 	    will be interpreted as "no limit".</li>

	 * </ul>

	 * </ul>

	 *

	 *

	 * <p><i><b>Important note:</b>

	 * <p><i><b>Important note:</b>

	 * 	To save performances, it is recommended to use BYTES limit rather than in rows. Indeed, the bytes limit can be taken

	 * 	To save performances, it is recommended to use BYTES limit rather than

	 * 	into account at directly when reading the bytes of the request, on the contrary of the rows limit which

	 * 	in rows. Indeed, the bytes limit can be taken into account at directly

	 * 	requires to parse the uploaded tables.

	 * 	when reading the bytes of the request, on the contrary of the rows limit

	 * 	which requires to parse the uploaded tables.

	 * </i></p>

	 * </i></p>

	 *

	 *

	 * @return	NULL if no limit must be set, or a two-items array ([0]: default value, [1]: maximum value).

	 * @return	NULL if no limit must be set,

	 *        	or a two-items array ([0]: default value, [1]: maximum value).

	 *

	 *

	 * @see #getUploadLimitType()

	 * @see #getUploadLimitType()

	 */

	 */

	public int[] getUploadLimit();

	public long[] getUploadLimit();





	/**

	/**

	 * <i>[OPTIONAL]</i>

	 * <i>[OPTIONAL]</i>

	 * <p>Get the type of each upload limit set by this service connection (and accessible with {@link #getUploadLimit()}).</p>

	 * <p>Get the type of each upload limit set by this service connection (and

	 * accessible with {@link #getUploadLimit()}).</p>

	 *

	 *

	 * <p><b>Important notes:</b></p>

	 * <p><b>Important notes:</b></p>

	 * <ul>

	 * <ul>

	 * 	<li><b>Exactly 2 values or a NULL object is expected here.</b></li>

	 * 	<li><b>Exactly 2 values or a NULL object is expected here.</b></li>

	 * 	<li><b>If NULL</b>, the upload limit will be considered as expressed in ROWS.</li>

	 * 	<li><b>If NULL</b>, the upload limit will be considered as expressed in

	 * 	<li><b>The structure of the object</b> returned by this function MUST be the same as the object returned by {@link #getUploadLimit()}.

	 * 	    ROWS.</li>

	 * 	    Particularly, the type given by the N-th item of this function must correspond to the N-th limit returned by {@link #getUploadLimit()}.</li>

	 * 	<li><b>The structure of the object</b> returned by this function MUST be

	 * 	    the same as the object returned by {@link #getUploadLimit()}.

	 * 	    Particularly, the type given by the N-th item of this function must

	 * 	    correspond to the N-th limit returned by {@link #getUploadLimit()}.

	 * 	</li>

	 * </ul>

	 * </ul>

	 *

	 *

	 * <p><i><b>Important note:</b>

	 * <p><i><b>Important note:</b>

	 * 	To save performances, it is recommended to use BYTES limit rather than in rows. Indeed, the bytes limit can be taken

	 * 	To save performances, it is recommended to use BYTES limit rather than

	 * 	into account at directly when reading the bytes of the request, on the contrary of the rows limit which

	 * 	in rows. Indeed, the bytes limit can be taken into account at directly

	 * 	requires to parse the uploaded tables.

	 * 	when reading the bytes of the request, on the contrary of the rows limit

	 * 	which requires to parse the uploaded tables.

	 * </i></p>

	 * </i></p>

	 *

	 *

	 * @return	NULL if limits should be expressed in ROWS, or a two-items array ([0]: type of getUploadLimit()[0], [1]: type of getUploadLimit()[1]). 

	 * @return	NULL if limits should be expressed in ROWS,

	 *        	or a two-items array ([0]: type of getUploadLimit()[0],

	 *        	[1]: type of getUploadLimit()[1]).

	 *

	 *

	 * @see #getUploadLimit()

	 * @see #getUploadLimit()

	 */

	 */
@@ -437,22 +493,19 @@ public interface ServiceConnection { 




	/**

	/**

	 * <i>[OPTIONAL]</i>

	 * <i>[OPTIONAL]</i>

	 * <p>Get the maximum size of the whole set of all tables uploaded in one request.

	 * <p>Get the maximum size of the whole set of all tables uploaded in one

	 * This size is expressed in bytes.</p>

	 * request. This size is expressed in bytes.</p>

	 *

	 *

	 * <p><b>IMPORTANT 1:

	 * <p><i><b>NOTE:</b>

	 * 	This value is always used when the upload capability is enabled.

	 * 	This value can be negative. In such case, there will be no limit on the

	 * </b></p>

	 * 	size of an HTTP request.

	 * 

	 * </i></p>

	 * <p><b>IMPORTANT 2:

	 * 	The value returned by this function MUST always be positive.

	 * 	A zero or negative value will throw an exception later while

	 * 	reading parameters in a request with some uploaded tables.

	 * </b></p>

	 *

	 *

	 * @return	A positive (&gt;0) value corresponding to the maximum number of bytes of all uploaded tables sent in one request.

	 * @return	A positive value (&gt;0) corresponding to the maximum number of

	 *        	bytes of all uploaded tables sent in one request.

	 *        	A negative value (&le;0) means "unlimited".

	 */

	 */

	public int getMaxUploadSize();

	public long getMaxUploadSize();





	/**

	/**

	 * <i><b>[MANDATORY]</b></i>

	 * <i><b>[MANDATORY]</b></i>

src/tap/TAPFactory.java

+13 −5
Original line number Original line Diff line number Diff line
@@ -16,7 +16,7 @@ package tap; 
 * You should have received a copy of the GNU Lesser General Public License

 * You should have received a copy of the GNU Lesser General Public License

 * along with TAPLibrary.  If not, see <http://www.gnu.org/licenses/>.

 * along with TAPLibrary.  If not, see <http://www.gnu.org/licenses/>.

 *

 *

 * Copyright 2012-2017 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 * Copyright 2012-2018 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 *                       Astronomisches Rechen Institut (ARI)

 *                       Astronomisches Rechen Institut (ARI)

 */

 */
@@ -29,6 +29,7 @@ import adql.parser.ADQLParser; 
import adql.parser.ADQLQueryFactory;

import adql.parser.ADQLQueryFactory;

import adql.parser.QueryChecker;

import adql.parser.QueryChecker;

import adql.query.ADQLQuery;

import adql.query.ADQLQuery;

import tap.ServiceConnection.LimitUnit;

import tap.db.DBConnection;

import tap.db.DBConnection;

import tap.metadata.TAPSchema;

import tap.metadata.TAPSchema;

import tap.parameters.TAPParameters;

import tap.parameters.TAPParameters;
@@ -63,7 +64,7 @@ import uws.service.request.RequestParser; 
 * </ul>

 * </ul>

 *

 *

 * @author Gr&eacute;gory Mantelet (CDS;ARI)

 * @author Gr&eacute;gory Mantelet (CDS;ARI)

 * @version 2.2 (09/2017)

 * @version 2.3 (09/2018)

 */

 */

public abstract class TAPFactory implements UWSFactory {

public abstract class TAPFactory implements UWSFactory {
@@ -485,7 +486,14 @@ public abstract class TAPFactory implements UWSFactory { 




	@Override

	@Override

	public RequestParser createRequestParser(final UWSFileManager fileManager) throws UWSException{

	public RequestParser createRequestParser(final UWSFileManager fileManager) throws UWSException{

		return new TAPRequestParser(fileManager);

		/* Fetch the maximum size for each uploaded file:

		 * ....only if set to a positive value and expressed in bytes. */

		long maxFileSize = -1;

		if (service.getUploadLimit() != null && service.getUploadLimit().length >= 2 && service.getUploadLimit()[1] > 0 && service.getUploadLimitType() != null && service.getUploadLimitType().length == service.getUploadLimit().length && service.getUploadLimitType()[1] != LimitUnit.rows){

			maxFileSize = service.getUploadLimit()[1] * service.getUploadLimitType()[1].bytesFactor();

		}

		// Finally create the appropriate RequestParser:

		return new TAPRequestParser(fileManager, service.uploadEnabled(), maxFileSize, service.getMaxUploadSize());

	}

	}





}
 
}

src/tap/TAPRequestParser.java

+69 −31
Original line number Original line Diff line number Diff line
@@ -44,18 +44,18 @@ import uws.service.request.UploadFile; 
 * </ul>

 * </ul>

 *

 *

 * <p>

 * <p>

 * 	The request body size is limited for the multipart. If you want to change

 * 	The request body size is limited for the multipart. You can change these

 * 	this limit, you MUST do it for each of these parsers, setting the following

 * 	limits with the constructor

 * 	static attributes: {@link MultipartParser#SIZE_LIMIT}.

 * 	{@link #TAPRequestParser(UWSFileManager, boolean, long, long)}

 * </p>

 * </p>

 *

 *

 * <p><i>Note:

 * <p><i>Note:

 * 	If you want to change the support other request parsing, you will have to

 * 	If you want to support other request parsing, you have to write your own

 * 	write your own {@link RequestParser} implementation.

 * 	{@link RequestParser} implementation.

 * </i></p>

 * </i></p>

 *

 *

 * @author Gr&eacute;gory Mantelet (ARI)

 * @author Gr&eacute;gory Mantelet (ARI;CDS)

 * @version 2.1 (06/2017)

 * @version 2.3 (09/2018)

 * @since 2.0

 * @since 2.0

 */

 */

public class TAPRequestParser implements RequestParser {

public class TAPRequestParser implements RequestParser {
@@ -76,6 +76,20 @@ public class TAPRequestParser implements RequestParser { 
	 * only when needed, by calling the function {@link #getMultipartParser()}. */

	 * only when needed, by calling the function {@link #getMultipartParser()}. */

	private RequestParser multipartParser = null;

	private RequestParser multipartParser = null;





	/** Indicates whether this parser should allow uploaded files.

	 * @since 2.3 */

	private final boolean allowUpload;



	/** Maximum size of a single uploaded file.

	 * <p><i><b>Note:</b> If negative (&le;0), "unlimited".</i></p>

	 * @since 2.3 */

	private final long maxFileSize;



	/** Maximum size of a whole HTTP Multipart Request.

	 * <p><i><b>Note:</b> If negative (&le;0), "unlimited".</i></p>

	 * @since 2.3 */

	private final long maxMultipartSize;



	/**

	/**

	 * Build a {@link RequestParser} able to choose the most appropriate

	 * Build a {@link RequestParser} able to choose the most appropriate

	 * {@link RequestParser} in function of the request content-type.

	 * {@link RequestParser} in function of the request content-type.
@@ -84,9 +98,32 @@ public class TAPRequestParser implements RequestParser { 
	 *                   	eventual upload. <b>MUST NOT be NULL</b>

	 *                   	eventual upload. <b>MUST NOT be NULL</b>

	 */

	 */

	public TAPRequestParser(final UWSFileManager fileManager){

	public TAPRequestParser(final UWSFileManager fileManager){

		this(fileManager, true, -1, -1);

	}



	/**

	 * Build a {@link RequestParser} able to choose the most appropriate

	 * {@link RequestParser} in function of the request content-type.

	 *

	 * @param fileManager		The file manager to use in order to store any

	 *                   		eventual upload. <b>MUST NOT be NULL</b>

	 * @param uploadEnabled		<code>true</code> to support UPLOADs,

	 *                     		<code>false</code> otherwise (i.e. all Multipart

	 *                     		requests with files will be rejected).

	 * @param maxFileSize		Maximum size for a single uploaded file.

	 *                   		<i>A negative value means "no limit".</i>

	 * @param maxRequestSize	Maximum size for a whole Multipart HTTP Request.

	 *                   		<i>A negative value means "no limit".</i>

	 *

	 * @since 2.3

	 */

	public TAPRequestParser(final UWSFileManager fileManager, final boolean uploadEnabled, final long maxFileSize, final long maxRequestSize){

		if (fileManager == null)

		if (fileManager == null)

			throw new NullPointerException("Missing file manager => can not create a TAPRequestParser!");

			throw new NullPointerException("Missing file manager => can not create a TAPRequestParser!");

		this.fileManager = fileManager;

		this.fileManager = fileManager;

		this.allowUpload = uploadEnabled;

		this.maxFileSize = maxFileSize;

		this.maxMultipartSize = maxRequestSize;

	}

	}





	@Override

	@Override
@@ -155,7 +192,7 @@ public class TAPRequestParser implements RequestParser { 
	 *        	requests. <i>Never NULL</i>

	 *        	requests. <i>Never NULL</i>

	 */

	 */

	private synchronized final RequestParser getMultipartParser(){

	private synchronized final RequestParser getMultipartParser(){

		return (multipartParser != null) ? multipartParser : (multipartParser = new MultipartParser(fileManager){

		return (multipartParser != null) ? multipartParser : (multipartParser = new MultipartParser(allowUpload, fileManager, maxFileSize, maxMultipartSize) {

			@Override

			@Override

			protected void consumeParameter(String name, Object value, final Map<String, Object> allParams){

			protected void consumeParameter(String name, Object value, final Map<String, Object> allParams){

				// Modify the value if it is an UPLOAD parameter:

				// Modify the value if it is an UPLOAD parameter:
@@ -167,7 +204,8 @@ public class TAPRequestParser implements RequestParser { 
					else if (name.equals(TAPJob.PARAM_UPLOAD) && value instanceof UploadFile){

					else if (name.equals(TAPJob.PARAM_UPLOAD) && value instanceof UploadFile){

						try{

						try{

							((UploadFile)value).deleteFile();

							((UploadFile)value).deleteFile();

						}catch(IOException ioe){}

						}catch(IOException ioe){

						}

						return;

						return;

					}

					}

					// use the same case for the parameter name:

					// use the same case for the parameter name: