Commit 16f7e8e3 authored by gmantele's avatar gmantele
Browse files

[TAP] Add the possibility to set the FetchSize on a DB Statement for the... 

[TAP] Add the possibility to set the FetchSize on a DB Statement for the execution of ADQL queries. It is also possible to set a fetchSize different for synchronous and asynchronous queries.
parent 3d796e0b

src/tap/ADQLExecutor.java

+12 −1
Original line number Diff line number Diff line
@@ -356,6 +356,9 @@ public class ADQLExecutor { 


			// 4. WRITE RESULT:

			startStep(ExecutionProgression.WRITING_RESULT);

			if (response != null && response.isCommitted())

				getLogger().logTAP(LogLevel.WARNING, report, "WRITING_RESULT", "HTTP request canceled/timeout! The result can not have been returned to the user.", null);

			else

				writeResult(queryResult);

			endStep();
@@ -548,6 +551,14 @@ public class ADQLExecutor { 
		// Log the start of execution:

		logger.logTAP(LogLevel.INFO, report, "EXECUTING", "Executing ADQL: " + adql.toADQL().replaceAll("(\t|\r?\n)+", " "), null);



		// Set the fetch size, if any:

		if (service.getFetchSize() != null && service.getFetchSize().length >= 1){

			if (report.synchronous && service.getFetchSize().length >= 2)

				dbConn.setFetchSize(service.getFetchSize()[1]);

			else

				dbConn.setFetchSize(service.getFetchSize()[0]);

		}



		// Execute the ADQL query:

		TableIterator result = dbConn.executeQuery(adql);

src/tap/ServiceConnection.java

+32 −2
Original line number Diff line number Diff line
@@ -16,13 +16,14 @@ package tap; 
 * You should have received a copy of the GNU Lesser General Public License

 * along with TAPLibrary.  If not, see <http://www.gnu.org/licenses/>.

 * 

 * Copyright 2012,2014 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 * Copyright 2012-2015 - UDS/Centre de Données astronomiques de Strasbourg (CDS),

 *                       Astronomisches Rechen Institut (ARI)

 */



import java.util.Collection;

import java.util.Iterator;



import tap.db.DBConnection;

import tap.formatter.OutputFormat;

import tap.log.DefaultTAPLog;

import tap.log.TAPLog;
@@ -42,7 +43,7 @@ import adql.db.FunctionDef; 
 * </p>

 * 

 * @author Gr&eacute;gory Mantelet (CDS;ARI)

 * @version 2.0 (01/2015)

 * @version 2.0 (03/2015)

 */

public interface ServiceConnection {
@@ -660,4 +661,33 @@ public interface ServiceConnection { 
	 */

	public OutputFormat getOutputFormat(final String mimeOrAlias);



	/**

	 * <i>[OPTIONAL]</i>

	 * <p>Get the size of result blocks to fetch from the database.</p>

	 * 

	 * <p>

	 * 	Rather than fetching a query result in a whole, it may be possible to specify to the database

	 * 	that results may be retrieved by blocks whose the size can be specified by this function.

	 * 	If supported by the DBMS and the JDBC driver, this feature may help sparing memory and avoid

	 * 	too much waiting time from the TAP /sync users (and thus, avoiding some HTTP client timeouts).

	 * </p>

	 * 

	 * <p><i>Note:

	 * 	Generally, this feature is well supported by DBMS. But for that, the used JDBC driver must use

	 * 	the V3 protocol. If anyway, this feature is supported neither by the DBMS, the JDBC driver nor your

	 * 	{@link DBConnection}, no error will be thrown if a value is returned by this function: it will be silently

	 * 	ignored by the library.

	 * </i></p>

	 * 

	 * @return	<i>null</i> or an array of 1 or 2 integers.

	 * 			If <i>null</i> (or empty array), no attempt to set fetch size will be done and so, ONLY the default

	 *        	value of the {@link DBConnection} will be used.

	 *        	[0]=fetchSize for async queries, [1]=fetchSize for sync queries.

	 *        	If [1] is omitted, it will be considered as equals to [0].

	 *        	If a fetchSize is negative or null, the default value of your JDBC driver will be used.

	 * 

	 * @since 2.0

	 */

	public int[] getFetchSize();



}

src/tap/config/ConfigurableServiceConnection.java

+46 −0
Original line number Diff line number Diff line
@@ -19,12 +19,15 @@ package tap.config; 
 * Copyright 2015 - Astronomisches Rechen Institut (ARI)

 */



import static tap.config.TAPConfiguration.DEFAULT_ASYNC_FETCH_SIZE;

import static tap.config.TAPConfiguration.DEFAULT_DIRECTORY_PER_USER;

import static tap.config.TAPConfiguration.DEFAULT_EXECUTION_DURATION;

import static tap.config.TAPConfiguration.DEFAULT_GROUP_USER_DIRECTORIES;

import static tap.config.TAPConfiguration.DEFAULT_MAX_ASYNC_JOBS;

import static tap.config.TAPConfiguration.DEFAULT_RETENTION_PERIOD;

import static tap.config.TAPConfiguration.DEFAULT_SYNC_FETCH_SIZE;

import static tap.config.TAPConfiguration.DEFAULT_UPLOAD_MAX_FILE_SIZE;

import static tap.config.TAPConfiguration.KEY_ASYNC_FETCH_SIZE;

import static tap.config.TAPConfiguration.KEY_COORD_SYS;

import static tap.config.TAPConfiguration.KEY_DEFAULT_EXECUTION_DURATION;

import static tap.config.TAPConfiguration.KEY_DEFAULT_OUTPUT_LIMIT;
@@ -47,6 +50,7 @@ import static tap.config.TAPConfiguration.KEY_MIN_LOG_LEVEL; 
import static tap.config.TAPConfiguration.KEY_OUTPUT_FORMATS;

import static tap.config.TAPConfiguration.KEY_PROVIDER_NAME;

import static tap.config.TAPConfiguration.KEY_SERVICE_DESCRIPTION;

import static tap.config.TAPConfiguration.KEY_SYNC_FETCH_SIZE;

import static tap.config.TAPConfiguration.KEY_TAP_FACTORY;

import static tap.config.TAPConfiguration.KEY_UDFS;

import static tap.config.TAPConfiguration.KEY_UPLOAD_ENABLED;
@@ -148,6 +152,8 @@ public final class ConfigurableServiceConnection implements ServiceConnection { 
	private LimitUnit[] uploadLimitTypes = new LimitUnit[2];

	private int maxUploadSize = DEFAULT_UPLOAD_MAX_FILE_SIZE;



	private int[] fetchSize = new int[]{DEFAULT_ASYNC_FETCH_SIZE,DEFAULT_SYNC_FETCH_SIZE};



	private UserIdentifier userIdentifier = null;



	private ArrayList<String> lstCoordSys = null;
@@ -191,6 +197,8 @@ public final class ConfigurableServiceConnection implements ServiceConnection { 
		addOutputFormats(tapConfig);

		// set output limits:

		initOutputLimits(tapConfig);

		// set fetch size:

		initFetchSize(tapConfig);



		// 7. CONFIGURE THE UPLOAD:

		// is upload enabled ?
@@ -466,6 +474,7 @@ public final class ConfigurableServiceConnection implements ServiceConnection { 
			outputFormats.add(new VOTableFormat(this, DataFormat.TABLEDATA));

			outputFormats.add(new VOTableFormat(this, DataFormat.FITS));

			outputFormats.add(new FITSFormat(this));

			outputFormats.add(new JSONFormat(this));

			outputFormats.add(new SVFormat(this, ",", true));

			outputFormats.add(new SVFormat(this, "\t", true));

			outputFormats.add(new TextFormat(this));
@@ -660,6 +669,38 @@ public final class ConfigurableServiceConnection implements ServiceConnection { 
			throw new TAPException("The default output limit (here: " + outputLimits[0] + ") MUST be less or equal to the maximum output limit (here: " + limit[0] + ")!");

	}



	private void initFetchSize(final Properties tapConfig) throws TAPException{

		fetchSize = new int[2];



		// Set the fetch size for asynchronous queries:

		String propVal = getProperty(tapConfig, KEY_ASYNC_FETCH_SIZE);

		if (propVal == null)

			fetchSize[0] = DEFAULT_ASYNC_FETCH_SIZE;

		else{

			try{

				fetchSize[0] = Integer.parseInt(propVal);

				if (fetchSize[0] < 0)

					fetchSize[0] = 0;

			}catch(NumberFormatException nfe){

				throw new TAPException("Integer expected for the property " + KEY_ASYNC_FETCH_SIZE + ": \"" + propVal + "\"!");

			}

		}



		// Set the fetch size for synchronous queries:

		propVal = getProperty(tapConfig, KEY_SYNC_FETCH_SIZE);

		if (propVal == null)

			fetchSize[1] = DEFAULT_SYNC_FETCH_SIZE;

		else{

			try{

				fetchSize[1] = Integer.parseInt(propVal);

				if (fetchSize[1] < 0)

					fetchSize[1] = 0;

			}catch(NumberFormatException nfe){

				throw new TAPException("Integer expected for the property " + KEY_SYNC_FETCH_SIZE + ": \"" + propVal + "\"!");

			}

		}

	}



	private void initUploadLimits(final Properties tapConfig) throws TAPException{

		Object[] limit = parseLimit(getProperty(tapConfig, KEY_DEFAULT_UPLOAD_LIMIT), KEY_DEFAULT_UPLOAD_LIMIT, true);

		uploadLimitTypes[0] = (LimitUnit)limit[1];
@@ -1182,4 +1223,9 @@ public final class ConfigurableServiceConnection implements ServiceConnection { 
		return udfs;

	}



	@Override

	public int[] getFetchSize(){

		return fetchSize;

	}



}

src/tap/config/TAPConfiguration.java

+4 −0
Original line number Diff line number Diff line
@@ -54,6 +54,10 @@ public final class TAPConfiguration { 
	public final static String KEY_SQL_TRANSLATOR = "sql_translator";

	public final static String VALUE_POSTGRESQL = "postgres";

	public final static String VALUE_PGSPHERE = "pgsphere";

	public final static String KEY_SYNC_FETCH_SIZE = "sync_fetch_size";

	public final static int DEFAULT_SYNC_FETCH_SIZE = 10000;

	public final static String KEY_ASYNC_FETCH_SIZE = "async_fetch_size";

	public final static int DEFAULT_ASYNC_FETCH_SIZE = 100000;

	public final static String KEY_DATASOURCE_JNDI_NAME = "datasource_jndi_name";

	public final static String KEY_JDBC_DRIVER = "jdbc_driver";

	public final static HashMap<String,String> VALUE_JDBC_DRIVERS = new HashMap<String,String>(4);

src/tap/config/tap_configuration_file.html

+31 −0
Original line number Diff line number Diff line
@@ -172,6 +172,37 @@ 
				</td>

				<td><ul><li>postgres</li><li>pgsphere</li><li>{apackage.MyADQLTranslator}</li></ul></td>

			</tr>

			<tr class="optional">

				<td class="done">sync_fetch_size</td>

				<td></td>

				<td>integer</td>

				<td>

					<p>Size of result blocks to fetch from the database when a ADQL query is executed in <em>Synchronous mode</em>.</p>

					<p>Rather than fetching a query result in a whole, it may be possible to specify to the database

	 				that results may be retrieved by blocks whose the size can be specified with this property.

	 				If supported by the DBMS and the JDBC driver, this feature may help sparing memory and avoid

	 				too much waiting time from the TAP /sync users (and thus, avoiding some HTTP client timeouts).</p>

	 				<p>A negative or null value means that the default value of the JDBC driver will be used. Generally,

	 				it means that the database must wait to have collected all data before sending them to the library.</p>

					<p><em>Default: <code>sync_fetch_size=10000</code></em></p>

				</td>

				<td><ul><li>10000 <em>(default)</em><li>0 <em>(wait for the the whole result)</em></li><li>100000</li></ul></td>

			</tr>

			<tr class="optional">

				<td class="done">async_fetch_size</td>

				<td></td>

				<td>integer</td>

				<td>

					<p>Size of result blocks to fetch from the database when an ADQL query is executed in <em>Asynchronous mode</em>.</p>

					<p>Rather than fetching a query result in a whole, it may be possible to specify to the database

	 				that results may be retrieved by blocks whose the size can be specified with this property.

	 				If supported by the DBMS and the JDBC driver, this feature may help sparing memory.</p>

	 				<p>A negative or null value means that the default value of the JDBC driver will be used. Generally,

	 				it means that the database must wait to have collected all data before sending them to the library.</p>

					<p><em>Default: <code>async_fetch_size=100000</code></em></p>

				</td>

				<td><ul><li>100000 <em>(default)</em><li>0 <em>(wait for the the whole result)</em></li><li>1000000</li></ul></td>

			</tr>

			

			<tr><td colspan="5">&#10551; JNDI datasource <i>(only if database_access=jndi)</i></td></tr>

			<tr class="mandatory">