ConfigurableServiceConnection.java

package tap.config;

/*
 * This file is part of TAPLibrary.
 *
 * TAPLibrary is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 *
 * TAPLibrary is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 *
 * You should have received a copy of the GNU Lesser General Public License
 * along with TAPLibrary.  If not, see <http://www.gnu.org/licenses/>.
 *
 * Copyright 2016-2020 - UDS/Centre de Données astronomiques de Strasbourg (CDS),
 *                       Astronomisches Rechen Institut (ARI)
 */

import static tap.config.TAPConfiguration.DEFAULT_ASYNC_FETCH_SIZE;
import static tap.config.TAPConfiguration.DEFAULT_DIRECTORY_PER_USER;
import static tap.config.TAPConfiguration.DEFAULT_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.DEFAULT_FIX_ON_FAIL;
import static tap.config.TAPConfiguration.DEFAULT_GROUP_USER_DIRECTORIES;
import static tap.config.TAPConfiguration.DEFAULT_LOGGER;
import static tap.config.TAPConfiguration.DEFAULT_MAX_ASYNC_JOBS;
import static tap.config.TAPConfiguration.DEFAULT_MAX_UPLOAD_LIMIT;
import static tap.config.TAPConfiguration.DEFAULT_RETENTION_PERIOD;
import static tap.config.TAPConfiguration.DEFAULT_SYNC_FETCH_SIZE;
import static tap.config.TAPConfiguration.DEFAULT_UPLOAD_MAX_REQUEST_SIZE;
import static tap.config.TAPConfiguration.KEY_ASYNC_FETCH_SIZE;
import static tap.config.TAPConfiguration.KEY_COORD_SYS;
import static tap.config.TAPConfiguration.KEY_DEFAULT_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.KEY_DEFAULT_OUTPUT_LIMIT;
import static tap.config.TAPConfiguration.KEY_DEFAULT_RETENTION_PERIOD;
import static tap.config.TAPConfiguration.KEY_DEFAULT_UPLOAD_LIMIT;
import static tap.config.TAPConfiguration.KEY_DIRECTORY_PER_USER;
import static tap.config.TAPConfiguration.KEY_FILE_MANAGER;
import static tap.config.TAPConfiguration.KEY_FILE_ROOT_PATH;
import static tap.config.TAPConfiguration.KEY_FIX_ON_FAIL;
import static tap.config.TAPConfiguration.KEY_GEOMETRIES;
import static tap.config.TAPConfiguration.KEY_GROUP_USER_DIRECTORIES;
import static tap.config.TAPConfiguration.KEY_LOGGER;
import static tap.config.TAPConfiguration.KEY_LOG_ROTATION;
import static tap.config.TAPConfiguration.KEY_MAX_ASYNC_JOBS;
import static tap.config.TAPConfiguration.KEY_MAX_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.KEY_MAX_OUTPUT_LIMIT;
import static tap.config.TAPConfiguration.KEY_MAX_RETENTION_PERIOD;
import static tap.config.TAPConfiguration.KEY_MAX_UPLOAD_LIMIT;
import static tap.config.TAPConfiguration.KEY_METADATA;
import static tap.config.TAPConfiguration.KEY_METADATA_FILE;
import static tap.config.TAPConfiguration.KEY_MIN_LOG_LEVEL;
import static tap.config.TAPConfiguration.KEY_OUTPUT_FORMATS;
import static tap.config.TAPConfiguration.KEY_PROVIDER_NAME;
import static tap.config.TAPConfiguration.KEY_SERVICE_DESCRIPTION;
import static tap.config.TAPConfiguration.KEY_SYNC_EXECUTION_DURATION;
import static tap.config.TAPConfiguration.KEY_SYNC_FETCH_SIZE;
import static tap.config.TAPConfiguration.KEY_TAP_FACTORY;
import static tap.config.TAPConfiguration.KEY_UDFS;
import static tap.config.TAPConfiguration.KEY_UPLOAD_ENABLED;
import static tap.config.TAPConfiguration.KEY_UPLOAD_MAX_FILE_SIZE;
import static tap.config.TAPConfiguration.KEY_UPLOAD_MAX_REQUEST_SIZE;
import static tap.config.TAPConfiguration.KEY_USER_IDENTIFIER;
import static tap.config.TAPConfiguration.SLF4J_LOGGER;
import static tap.config.TAPConfiguration.VALUE_ALL;
import static tap.config.TAPConfiguration.VALUE_ANY;
import static tap.config.TAPConfiguration.VALUE_CSV;
import static tap.config.TAPConfiguration.VALUE_DB;
import static tap.config.TAPConfiguration.VALUE_FITS;
import static tap.config.TAPConfiguration.VALUE_HTML;
import static tap.config.TAPConfiguration.VALUE_JSON;
import static tap.config.TAPConfiguration.VALUE_LOCAL;
import static tap.config.TAPConfiguration.VALUE_NONE;
import static tap.config.TAPConfiguration.VALUE_SV;
import static tap.config.TAPConfiguration.VALUE_TEXT;
import static tap.config.TAPConfiguration.VALUE_TSV;
import static tap.config.TAPConfiguration.VALUE_VOT;
import static tap.config.TAPConfiguration.VALUE_VOTABLE;
import static tap.config.TAPConfiguration.VALUE_XML;
import static tap.config.TAPConfiguration.fetchClass;
import static tap.config.TAPConfiguration.getProperty;
import static tap.config.TAPConfiguration.hasConstructor;
import static tap.config.TAPConfiguration.isClassName;
import static tap.config.TAPConfiguration.newInstance;
import static tap.config.TAPConfiguration.parseLimit;

import java.io.File;
import java.io.IOException;
import java.lang.reflect.Constructor;
import java.lang.reflect.InvocationTargetException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.Iterator;
import java.util.Properties;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import adql.db.FunctionDef;
import adql.db.STCS;
import adql.parser.ParseException;
import adql.query.operand.function.UserDefinedFunction;
import tap.ServiceConnection;
import tap.TAPException;
import tap.TAPFactory;
import tap.db.DBConnection;
import tap.db.JDBCConnection;
import tap.db.QueryExecutor;
import tap.formatter.FITSFormat;
import tap.formatter.HTMLFormat;
import tap.formatter.JSONFormat;
import tap.formatter.OutputFormat;
import tap.formatter.SVFormat;
import tap.formatter.TextFormat;
import tap.formatter.VOTableFormat;
import tap.log.DefaultTAPLog;
import tap.log.Slf4jTAPLog;
import tap.log.TAPLog;
import tap.metadata.TAPMetadata;
import tap.metadata.TableSetParser;
import uk.ac.starlink.votable.DataFormat;
import uk.ac.starlink.votable.VOTableVersion;
import uws.UWSException;
import uws.service.UserIdentifier;
import uws.service.file.LocalUWSFileManager;
import uws.service.file.UWSFileManager;
import uws.service.log.UWSLog.LogLevel;
import static tap.config.TAPConfiguration.KEY_QUERY_EXECUTOR;

/**
 * Concrete implementation of {@link ServiceConnection}, fully parameterized
 * with a TAP configuration file.
 *
 * <p>
 * 	Every aspects of the TAP service are configured here. This instance is also
 * 	creating the {@link TAPFactory} using the TAP configuration file thanks to
 * 	the implementation {@link ConfigurableTAPFactory}.
 * </p>
 *
 * @author Gr&eacute;gory Mantelet (CDS;ARI)
 * @version 2.4 (08/2020)
 * @since 2.0
 */
public final class ConfigurableServiceConnection implements ServiceConnection {

	/** File manager to use in the TAP service. */
	private UWSFileManager fileManager;

	/** Object to use in the TAP service in order to log different types of
	 * messages (e.g. DEBUG, INFO, WARNING, ERROR, FATAL). */
	private TAPLog logger;

	/** Factory which can create different types of objects for the TAP service
	 * (e.g. database connection). */
	private TAPFactory tapFactory;

	/** Object gathering all metadata of this TAP service. */
	private final TAPMetadata metadata;

	/** Name of the organization/person providing the TAP service.  */
	private final String providerName;
	/** Description of the TAP service. */
	private final String serviceDescription;

	/** Indicate whether the TAP service is available or not. */
	private boolean isAvailable = false;	// the TAP service must be disabled until the end of its connection initialization
	/** Description of the available or unavailable state of the TAP service. */
	private String availability = "TAP service not yet initialized.";

	/** Maximum number of asynchronous jobs that can run simultaneously. */
	private int maxAsyncJobs = DEFAULT_MAX_ASYNC_JOBS;

	/** Array of 3 integers: resp. default, maximum and sync. execution
	 * durations. <em>All durations are expressed in milliseconds.</em> */
	private int[] executionDuration = new int[3];
	/** Array of 2 integers: resp. default and maximum retention period.
	 * <em>Both period are expressed in seconds.</em> */
	private int[] retentionPeriod = new int[2];

	/** List of all available output formatters. */
	private final ArrayList<OutputFormat> outputFormats;

	/** Array of 2 integers: resp. default and maximum output limit.
	 * <em>Each limit is expressed in a unit specified in the array
	 * {@link #outputLimitTypes}.</em> */
	private int[] outputLimits = new int[]{ -1, -1 };
	/** Array of 2 limit units: resp. unit of the default output limit and unit
	 * of the maximum output limit. */
	private LimitUnit[] outputLimitTypes = new LimitUnit[2];

	/** Indicate whether the UPLOAD feature is enabled or not. */
	private boolean isUploadEnabled = false;
	/** Array of 2 integers: resp. default and maximum upload limit.
	 * <p><em>Each limit is expressed in a unit specified in the array
	 * {@link #uploadLimitTypes}.</em></p> */
	private long[] uploadLimits = new long[]{ -1L, -1L };
	/** Array of 2 limit units: resp. unit of the default upload limit and unit
	 * of the maximum upload limit. */
	private LimitUnit[] uploadLimitTypes = new LimitUnit[2];
	/** The maximum size of a set of uploaded files.
	 * <p><em>This size is expressed in bytes.</em></p> */
	private long maxUploadSize = DEFAULT_UPLOAD_MAX_REQUEST_SIZE;

	/** Array of 2 integers: resp. default and maximum fetch size.
	 * <em>Both sizes are expressed in number of rows.</em> */
	private int[] fetchSize = new int[]{ DEFAULT_ASYNC_FETCH_SIZE, DEFAULT_SYNC_FETCH_SIZE };

	/** The method to use in order to identify a TAP user. */
	private UserIdentifier userIdentifier = null;
        
        private QueryExecutor queryExecutor = null;

	/** List of all allowed coordinate systems.
	 * <em>
	 * 	If NULL, all coord. sys. are allowed. If empty list, none is allowed.
	 * </em> */
	private ArrayList<String> lstCoordSys = null;

	/** List of all allowed ADQL geometrical functions.
	 * <em>
	 * 	If NULL, all geometries are allowed. If empty list, none is allowed.
	 * </em> */
	private ArrayList<String> geometries = null;
	private final String GEOMETRY_REGEXP = "(AREA|BOX|CENTROID|CIRCLE|CONTAINS|DISTANCE|COORD1|COORD2|COORDSYS|INTERSECTS|POINT|POLYGON|REGION)";

	/** List of all known and allowed User Defined Functions.
	 * <em>If NULL, any unknown function is allowed. If empty list, none is
	 * allowed.</em> */
	private Collection<FunctionDef> udfs = new ArrayList<FunctionDef>(0);

	/** Indicate whether the input ADQL query should be automatically fixed
	 * if its parsing fails because of an incorrect tokenization.
	 * @since 2.3 */
	private boolean isFixOnFailEnabled = DEFAULT_FIX_ON_FAIL;

	/**
	 * Create a TAP service description thanks to the given TAP configuration
	 * file.
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws NullPointerException	If the given properties set is NULL.
	 * @throws TAPException			If a property is wrong or missing.
	 */
	public ConfigurableServiceConnection(final Properties tapConfig) throws NullPointerException, TAPException {
		this(tapConfig, null);
	}

	/**
	 * Create a TAP service description thanks to the given TAP configuration
	 * file.
	 *
	 * @param tapConfig		The content of the TAP configuration file.
	 * @param webAppRootDir	The directory of the Web Application running this
	 *                     	TAP service. <em>In this directory another directory
	 *                     	may be created in order to store all TAP service
	 *                     	files if none is specified in the given TAP
	 *                     	configuration file.</em>
	 *
	 * @throws NullPointerException	If the given properties set is NULL.
	 * @throws TAPException			If a property is wrong or missing.
	 */
	public ConfigurableServiceConnection(final Properties tapConfig, final String webAppRootDir) throws NullPointerException, TAPException {
		if (tapConfig == null)
			throw new NullPointerException("Missing TAP properties! ");

		// 1. INITIALIZE THE FILE MANAGER:
		initFileManager(tapConfig, webAppRootDir);

		// 2. CREATE THE LOGGER:
		initLogger(tapConfig);

		// 3. BUILD THE TAP FACTORY:
		initFactory(tapConfig);

		// 4. GET THE METADATA:
		metadata = initMetadata(tapConfig, webAppRootDir);

		// 6. SET ALL GENERAL SERVICE CONNECTION INFORMATION:
		providerName = getProperty(tapConfig, KEY_PROVIDER_NAME);
		serviceDescription = getProperty(tapConfig, KEY_SERVICE_DESCRIPTION);
		initMaxAsyncJobs(tapConfig);
		initRetentionPeriod(tapConfig);
		initExecutionDuration(tapConfig);

		// 7. CONFIGURE OUTPUT:
		// default output format = VOTable:
		outputFormats = new ArrayList<OutputFormat>(1);
		// set output formats:
		addOutputFormats(tapConfig);
		// set output limits:
		initOutputLimits(tapConfig);
		// set fetch size:
		initFetchSize(tapConfig);

		// 8. CONFIGURE THE UPLOAD:
		// is upload enabled ?
		isUploadEnabled = Boolean.parseBoolean(getProperty(tapConfig, KEY_UPLOAD_ENABLED));
		// set upload limits:
		initUploadLimits(tapConfig);
		// set the maximum upload file size:
		initMaxUploadSize(tapConfig);

		// 9. SET A USER IDENTIFIER:
		initUserIdentifier(tapConfig);

                // 10. SET A QUERY EXECUTOR:
                initQueryExecutor(tapConfig);
                
		// 11. CONFIGURE ADQL:
		initCoordSys(tapConfig);
		initADQLGeometries(tapConfig);
		initUDFs(tapConfig);
		isFixOnFailEnabled = Boolean.parseBoolean(getProperty(tapConfig, KEY_FIX_ON_FAIL));
	}

	/**
	 * Initialize the management of TAP service files using the given TAP configuration file.
	 *
	 * @param tapConfig		The content of the TAP configuration file.
	 * @param webAppRootDir	The directory of the Web Application running this TAP service.
	 *                     	<em>This directory may be used only to search the root TAP directory
	 *                     	if specified with a relative path in the TAP configuration file.</em>
	 *
	 * @throws TAPException	If a property is wrong or missing, or if an error occurs while creating the file manager.
	 */
	private void initFileManager(final Properties tapConfig, final String webAppRootDir) throws TAPException {
		// Read the desired file manager:
		String fileManagerType = getProperty(tapConfig, KEY_FILE_MANAGER);
		if (fileManagerType == null)
			throw new TAPException("The property \"" + KEY_FILE_MANAGER + "\" is missing! It is required to create a TAP Service. Two possible values: " + VALUE_LOCAL + " or a class name between {...}.");
		else
			fileManagerType = fileManagerType.trim();

		// LOCAL file manager:
		if (fileManagerType.equalsIgnoreCase(VALUE_LOCAL)) {
			// Read the desired root path:
			String rootPath = getProperty(tapConfig, KEY_FILE_ROOT_PATH);
			if (rootPath == null)
				throw new TAPException("The property \"" + KEY_FILE_ROOT_PATH + "\" is missing! It is required to create a TAP Service. Please provide a path toward a directory which will contain all files related to the service.");
			File rootFile = getFile(rootPath, webAppRootDir, KEY_FILE_ROOT_PATH);

			// Determine whether there should be one directory for each user:
			String propValue = getProperty(tapConfig, KEY_DIRECTORY_PER_USER);
			boolean oneDirectoryPerUser = (propValue == null) ? DEFAULT_DIRECTORY_PER_USER : Boolean.parseBoolean(propValue);

			// Determine whether there should be one directory for each user:
			propValue = getProperty(tapConfig, KEY_GROUP_USER_DIRECTORIES);
			boolean groupUserDirectories = (propValue == null) ? DEFAULT_GROUP_USER_DIRECTORIES : Boolean.parseBoolean(propValue);

			// Build the Local TAP File Manager:
			try {
				fileManager = new LocalUWSFileManager(rootFile, oneDirectoryPerUser, groupUserDirectories);
			} catch(UWSException e) {
				throw new TAPException("The property \"" + KEY_FILE_ROOT_PATH + "\" (" + rootPath + ") is incorrect: " + e.getMessage());
			}
		}
		// CUSTOM file manager:
		else
			fileManager = newInstance(fileManagerType, KEY_FILE_MANAGER, UWSFileManager.class, new Class<?>[]{ Properties.class }, new Object[]{ tapConfig });
	}

	/**
	 * <p>Resolve the given file name/path.</p>
	 *
	 * <p>
	 * 	If not an absolute path, the given path may be either relative or absolute. A relative path is always considered
	 * 	as relative from the Web Application directory (supposed to be given in 2nd parameter).
	 * </p>
	 *
	 * @param filePath			Path/Name of the file to get.
	 * @param webAppRootPath	Web Application directory local path.
	 * @param propertyName		Name of the property which gives the given file path.
	 *
	 * @return	The specified File instance.
	 *
	 * @throws ParseException	If the given file path is a URI/URL.
	 */
	protected static final File getFile(final String filePath, final String webAppRootPath, final String propertyName) throws TAPException {
		if (filePath == null)
			return null;
		else if (filePath.matches(".*:.*"))
			throw new TAPException("Incorrect file path for the property \"" + propertyName + "\": \"" + filePath + "\"! URI/URLs are not expected here.");

		File f = new File(filePath);
		if (f.isAbsolute())
			return f;
		else
			return new File(webAppRootPath, filePath);
	}

	/**
	 * Initialise the TAP logger with the given TAP configuration file.
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws TAPException	If no instance of the specified custom logger can
	 *                     	be created.
	 */
	private void initLogger(final Properties tapConfig) throws TAPException {
		// Create the logger:
		String propValue = getProperty(tapConfig, KEY_LOGGER);
		if (propValue == null || propValue.trim().equalsIgnoreCase(DEFAULT_LOGGER))
			logger = new DefaultTAPLog(fileManager);
		else if (propValue == null || propValue.trim().equalsIgnoreCase(SLF4J_LOGGER))
			logger = new Slf4jTAPLog();
		else
			logger = newInstance(propValue, KEY_LOGGER, TAPLog.class, new Class<?>[]{ UWSFileManager.class }, new Object[]{ fileManager });

		// Set some options for the default logger:
		if (propValue == null || propValue.trim().equalsIgnoreCase(DEFAULT_LOGGER)) {

			// Set the minimum log level:
			propValue = getProperty(tapConfig, KEY_MIN_LOG_LEVEL);
			if (propValue != null) {
				try {
					((DefaultTAPLog)logger).setMinLogLevel(LogLevel.valueOf(propValue.toUpperCase()));
				} catch(IllegalArgumentException iae) {
				}
			}

			// Set the log rotation period, if any:
			if (fileManager instanceof LocalUWSFileManager) {
				propValue = getProperty(tapConfig, KEY_LOG_ROTATION);
				if (propValue != null)
					((LocalUWSFileManager)fileManager).setLogRotationFreq(propValue);
			}
		}

		// Log the successful initialisation of the logger:
		logger.info("Logger initialized - {" + logger.getConfigString() + "}");
	}

	/**
	 * <p>Initialize the {@link TAPFactory} to use.</p>
	 *
	 * <p>
	 * 	The built factory is either a {@link ConfigurableTAPFactory} instance (by default) or
	 * 	an instance of the class specified in the TAP configuration file.
	 * </p>
	 *
	 * @param tapConfig		The content of the TAP configuration file.
	 *
	 * @throws TAPException	If an error occurs while building the specified {@link TAPFactory}.
	 *
	 * @see ConfigurableTAPFactory
	 */
	private void initFactory(final Properties tapConfig) throws TAPException {
		String propValue = getProperty(tapConfig, KEY_TAP_FACTORY);
		if (propValue == null)
			tapFactory = new ConfigurableTAPFactory(this, tapConfig);
		else if (hasConstructor(propValue, KEY_TAP_FACTORY, TAPFactory.class, new Class<?>[]{ ServiceConnection.class, Properties.class }))
			tapFactory = newInstance(propValue, KEY_TAP_FACTORY, TAPFactory.class, new Class<?>[]{ ServiceConnection.class, Properties.class }, new Object[]{ this, tapConfig });
		else
			tapFactory = newInstance(propValue, KEY_TAP_FACTORY, TAPFactory.class, new Class<?>[]{ ServiceConnection.class }, new Object[]{ this });
	}

	/**
	 * Initialize the TAP metadata (i.e. database schemas, tables and columns and their attached metadata).
	 *
	 * @param tapConfig		The content of the TAP configuration file.
	 * @param webAppRootDir	Web Application directory local path.
	 *                     	<em>This directory may be used if a relative path is given for an XML metadata file.</em>
	 *
	 * @return	The extracted TAP metadata.
	 *
	 * @throws TAPException	If some TAP configuration file properties are wrong or missing,
	 *                     	or if an error has occurred while extracting the metadata from the database or the XML file.
	 *
	 * @see DBConnection#getTAPSchema()
	 * @see TableSetParser
	 */
	private TAPMetadata initMetadata(final Properties tapConfig, final String webAppRootDir) throws TAPException {
		// Get the fetching method to use:
		String metaFetchType = getProperty(tapConfig, KEY_METADATA);
		if (metaFetchType == null)
			throw new TAPException("The property \"" + KEY_METADATA + "\" is missing! It is required to create a TAP Service. Three possible values: " + VALUE_XML + " (to get metadata from a TableSet XML document), " + VALUE_DB + " (to fetch metadata from the database schema TAP_SCHEMA) or the name (between {}) of a class extending TAPMetadata. Only " + VALUE_XML + " and " + VALUE_DB + " can be followed by the path of a class extending TAPMetadata.");

		// Extract a custom class suffix if any for XML and DB options:
		String customMetaClass = null;
		if (metaFetchType.toLowerCase().matches("(" + VALUE_XML + "|" + VALUE_DB + ").*")) {
			int indSep = metaFetchType.toLowerCase().startsWith(VALUE_XML) ? 3 : 2;
			customMetaClass = metaFetchType.substring(indSep).trim();
			metaFetchType = metaFetchType.substring(0, indSep);
			if (customMetaClass.length() == 0)
				customMetaClass = null;
			else if (!isClassName(customMetaClass))
				throw new TAPException("Unexpected string after the fetching method \"" + metaFetchType + "\": \"" + customMetaClass + "\"! The full name of a class extending TAPMetadata was expected. If it is a class name, then it must be specified between {}.");
		}

		TAPMetadata metadata = null;

		// GET METADATA FROM XML & UPDATE THE DATABASE (schema TAP_SCHEMA only):
		if (metaFetchType.equalsIgnoreCase(VALUE_XML)) {
			// Get the XML file path:
			String xmlFilePath = getProperty(tapConfig, KEY_METADATA_FILE);
			if (xmlFilePath == null)
				throw new TAPException("The property \"" + KEY_METADATA_FILE + "\" is missing! According to the property \"" + KEY_METADATA + "\", metadata must be fetched from an XML document. The local file path of it MUST be provided using the property \"" + KEY_METADATA_FILE + "\".");

			// Parse the XML document and build the corresponding metadata:
			try {
				metadata = (new TableSetParser()).parse(getFile(xmlFilePath, webAppRootDir, KEY_METADATA_FILE));
			} catch(IOException ioe) {
				throw new TAPException("A grave error occurred while reading/parsing the TableSet XML document: \"" + xmlFilePath + "\"!", ioe);
			}

			// Update the database:
			DBConnection conn = null;
			try {
				conn = tapFactory.getConnection("SET_TAP_SCHEMA");
				conn.setTAPSchema(metadata);
			} finally {
				if (conn != null)
					tapFactory.freeConnection(conn);
			}
		}
		// GET METADATA FROM DATABASE (schema TAP_SCHEMA):
		else if (metaFetchType.equalsIgnoreCase(VALUE_DB)) {
			DBConnection conn = null;
			try {
				// get a db connection:
				conn = tapFactory.getConnection("GET_TAP_SCHEMA");

				// fetch and set the ADQL<->DB mapping for all standard TAP_SCHEMA items:
				if (conn instanceof JDBCConnection) {
					HashMap<String, String> dbMapping = new HashMap<String, String>(10);
					// fetch the mapping from the Property file:
					for(String key : tapConfig.stringPropertyNames()) {
						if (key.trim().startsWith("TAP_SCHEMA") && tapConfig.getProperty(key) != null && tapConfig.getProperty(key).trim().length() > 0)
							dbMapping.put(key.trim(), tapConfig.getProperty(key));
					}
					// set the mapping into the DB connection:
					((JDBCConnection)conn).setDBMapping(dbMapping);
				}

				// fetch TAP_SCHEMA:
				metadata = conn.getTAPSchema();
			} finally {
				if (conn != null)
					tapFactory.freeConnection(conn);
			}
		}
		// MANUAL ~ TAPMETADATA CLASS
		else if (isClassName(metaFetchType)) {
			/* 1. Get the metadata */
			// get the class:
			Class<? extends TAPMetadata> metaClass = fetchClass(metaFetchType, KEY_METADATA, TAPMetadata.class);
			if (metaClass == TAPMetadata.class)
				throw new TAPException("Wrong class for the property \"" + KEY_METADATA + "\": \"" + metaClass.getName() + "\"! The class provided in this property MUST EXTEND tap.metadata.TAPMetadata.");
			try {
				// get one of the expected constructors:
				try {
					// (UWSFileManager, TAPFactory, TAPLog):
					Constructor<? extends TAPMetadata> constructor = metaClass.getConstructor(UWSFileManager.class, TAPFactory.class, TAPLog.class);
					// create the TAP metadata:
					metadata = constructor.newInstance(fileManager, tapFactory, logger);
				} catch(NoSuchMethodException nsme) {
					// () (empty constructor):
					Constructor<? extends TAPMetadata> constructor = metaClass.getConstructor();
					// create the TAP metadata:
					metadata = constructor.newInstance();
				}
			} catch(NoSuchMethodException nsme) {
				throw new TAPException("Missing constructor tap.metadata.TAPMetadata() or tap.metadata.TAPMetadata(uws.service.file.UWSFileManager, tap.TAPFactory, tap.log.TAPLog)! See the value \"" + metaFetchType + "\" of the property \"" + KEY_METADATA + "\".");
			} catch(InstantiationException ie) {
				throw new TAPException("Impossible to create an instance of an abstract class: \"" + metaClass.getName() + "\"! See the value \"" + metaFetchType + "\" of the property \"" + KEY_METADATA + "\".");
			} catch(InvocationTargetException ite) {
				if (ite.getCause() != null) {
					if (ite.getCause() instanceof TAPException)
						throw (TAPException)ite.getCause();
					else
						throw new TAPException(ite.getCause());
				} else
					throw new TAPException(ite);
			} catch(Exception ex) {
				throw new TAPException("Impossible to create an instance of tap.metadata.TAPMetadata as specified in the property \"" + KEY_METADATA + "\": \"" + metaFetchType + "\"!", ex);
			}

			/* 2. Update the database */
			DBConnection conn = null;
			try {
				conn = tapFactory.getConnection("SET_TAP_SCHEMA");
				conn.setTAPSchema(metadata);
			} finally {
				if (conn != null)
					tapFactory.freeConnection(conn);
			}
		}
		// INCORRECT VALUE => ERROR!
		else
			throw new TAPException("Unsupported value for the property \"" + KEY_METADATA + "\": \"" + metaFetchType + "\"! Only two values are allowed: " + VALUE_XML + " (to get metadata from a TableSet XML document) or " + VALUE_DB + " (to fetch metadata from the database schema TAP_SCHEMA). Only " + VALUE_XML + " and " + VALUE_DB + " can be followed by the path of a class extending TAPMetadata.");

		// Create the custom TAPMetadata extension if any is provided (THEORETICALLY, JUST FOR XML and DB):
		if (customMetaClass != null) {
			// get the class:
			Class<? extends TAPMetadata> metaClass = fetchClass(customMetaClass, KEY_METADATA, TAPMetadata.class);
			if (metaClass == TAPMetadata.class)
				throw new TAPException("Wrong class for the property \"" + KEY_METADATA + "\": \"" + metaClass.getName() + "\"! The class provided in this property MUST EXTEND tap.metadata.TAPMetadata.");
			try {
				// get one of the expected constructors:
				try {
					// (TAPMetadata, UWSFileManager, TAPFactory, TAPLog):
					Constructor<? extends TAPMetadata> constructor = metaClass.getConstructor(TAPMetadata.class, UWSFileManager.class, TAPFactory.class, TAPLog.class);
					// create the TAP metadata:
					metadata = constructor.newInstance(metadata, fileManager, tapFactory, logger);
				} catch(NoSuchMethodException nsme) {
					// (TAPMetadata):
					Constructor<? extends TAPMetadata> constructor = metaClass.getConstructor(TAPMetadata.class);
					// create the TAP metadata:
					metadata = constructor.newInstance(metadata);
				}
			} catch(NoSuchMethodException nsme) {
				throw new TAPException("Missing constructor by copy tap.metadata.TAPMetadata(tap.metadata.TAPMetadata) or tap.metadata.TAPMetadata(tap.metadata.TAPMetadata, uws.service.file.UWSFileManager, tap.TAPFactory, tap.log.TAPLog)! See the value \"" + metaFetchType + "\" of the property \"" + KEY_METADATA + "\".");
			} catch(InstantiationException ie) {
				throw new TAPException("Impossible to create an instance of an abstract class: \"" + metaClass.getName() + "\"! See the value \"" + metaFetchType + "\" of the property \"" + KEY_METADATA + "\".");
			} catch(InvocationTargetException ite) {
				if (ite.getCause() != null) {
					if (ite.getCause() instanceof TAPException)
						throw (TAPException)ite.getCause();
					else
						throw new TAPException(ite.getCause());
				} else
					throw new TAPException(ite);
			} catch(Exception ex) {
				throw new TAPException("Impossible to create an instance of tap.metadata.TAPMetadata as specified in the property \"" + KEY_METADATA + "\": \"" + metaFetchType + "\"!", ex);
			}
		}

		return metadata;
	}

	/**
	 * Initialize the maximum number of asynchronous jobs.
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws TAPException	If the corresponding TAP configuration property is wrong.
	 */
	private void initMaxAsyncJobs(final Properties tapConfig) throws TAPException {
		// Get the property value:
		String propValue = getProperty(tapConfig, KEY_MAX_ASYNC_JOBS);
		try {
			// If a value is provided, cast it into an integer and set the attribute:
			maxAsyncJobs = (propValue == null) ? DEFAULT_MAX_ASYNC_JOBS : Integer.parseInt(propValue);
		} catch(NumberFormatException nfe) {
			throw new TAPException("Integer expected for the property \"" + KEY_MAX_ASYNC_JOBS + "\", instead of: \"" + propValue + "\"!");
		}
	}

	/**
	 * Initialize the default and maximum retention period.
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws TAPException	If the corresponding TAP configuration properties are wrong.
	 */
	private void initRetentionPeriod(final Properties tapConfig) throws TAPException {
		retentionPeriod = new int[2];

		// Set the default period:
		String propValue = getProperty(tapConfig, KEY_DEFAULT_RETENTION_PERIOD);
		try {
			retentionPeriod[0] = (propValue == null) ? DEFAULT_RETENTION_PERIOD : Integer.parseInt(propValue);
		} catch(NumberFormatException nfe) {
			throw new TAPException("Integer expected for the property \"" + KEY_DEFAULT_RETENTION_PERIOD + "\", instead of: \"" + propValue + "\"!");
		}

		// Set the maximum period:
		propValue = getProperty(tapConfig, KEY_MAX_RETENTION_PERIOD);
		try {
			retentionPeriod[1] = (propValue == null) ? DEFAULT_RETENTION_PERIOD : Integer.parseInt(propValue);
		} catch(NumberFormatException nfe) {
			throw new TAPException("Integer expected for the property \"" + KEY_MAX_RETENTION_PERIOD + "\", instead of: \"" + propValue + "\"!");
		}

		// The maximum period MUST be greater or equals than the default period.
		// If not, the default period is set (so decreased) to the maximum period.
		if (retentionPeriod[1] > 0 && retentionPeriod[1] < retentionPeriod[0])
			retentionPeriod[0] = retentionPeriod[1];
	}

	/**
	 * Initialize the default and maximum execution duration.
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws TAPException	If the corresponding TAP configuration properties are wrong.
	 */
	private void initExecutionDuration(final Properties tapConfig) throws TAPException {
		executionDuration = new int[3];

		// Set the default duration:
		String propValue = getProperty(tapConfig, KEY_DEFAULT_EXECUTION_DURATION);
		try {
			executionDuration[0] = (propValue == null) ? DEFAULT_EXECUTION_DURATION : Integer.parseInt(propValue);
		} catch(NumberFormatException nfe) {
			throw new TAPException("Integer expected for the property \"" + KEY_DEFAULT_EXECUTION_DURATION + "\", instead of: \"" + propValue + "\"!");
		}

		// Set the maximum duration:
		propValue = getProperty(tapConfig, KEY_MAX_EXECUTION_DURATION);
		try {
			executionDuration[1] = (propValue == null) ? DEFAULT_EXECUTION_DURATION : Integer.parseInt(propValue);
		} catch(NumberFormatException nfe) {
			throw new TAPException("Integer expected for the property \"" + KEY_MAX_EXECUTION_DURATION + "\", instead of: \"" + propValue + "\"!");
		}

		// Set the synchronous duration:
		propValue = getProperty(tapConfig, KEY_SYNC_EXECUTION_DURATION);
		try {
			executionDuration[2] = (propValue == null) ? DEFAULT_EXECUTION_DURATION : Integer.parseInt(propValue);
		} catch(NumberFormatException nfe) {
			throw new TAPException("Integer expected for the property \"" + KEY_SYNC_EXECUTION_DURATION + "\", instead of: \"" + propValue + "\"!");
		}

		/* The maximum duration MUST be greater or equals than the default
		 * duration. If not, the default duration is set (so decreased) to the
		 * maximum duration: */
		if (executionDuration[1] > 0 && executionDuration[1] < executionDuration[0])
			executionDuration[0] = executionDuration[1];

		/* The synchronous duration MUST be less or equals than the default
		 * duration (or the max if no default is set). If not, the sync.
		 * duration is set (so decreased) to the default duration (or max if no
		 * default): */
		if (executionDuration[0] > 0 && executionDuration[0] < executionDuration[2])
			executionDuration[2] = executionDuration[0];
		else if (executionDuration[0] <= 0 && executionDuration[1] > 0 && executionDuration[1] < executionDuration[2])
			executionDuration[2] = executionDuration[1];
	}

	/**
	 * <p>Initialize the list of all output format that the TAP service must support.</p>
	 *
	 * <p>
	 * 	This function ensures that at least one VOTable format is part of the returned list,
	 * 	even if none has been specified in the TAP configuration file. Indeed, the VOTable format is the only
	 * 	format required for a TAP service.
	 * </p>
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws TAPException	If the corresponding TAP configuration properties are wrong.
	 */
	private void addOutputFormats(final Properties tapConfig) throws TAPException {
		// Fetch the value of the property for additional output formats:
		String formats = getProperty(tapConfig, KEY_OUTPUT_FORMATS);

		// SPECIAL VALUE "ALL":
		if (formats == null || formats.equalsIgnoreCase(VALUE_ALL)) {
			outputFormats.add(new VOTableFormat(this, DataFormat.BINARY));
			outputFormats.add(new VOTableFormat(this, DataFormat.BINARY2));
			outputFormats.add(new VOTableFormat(this, DataFormat.TABLEDATA));
			outputFormats.add(new VOTableFormat(this, DataFormat.FITS));
			outputFormats.add(new FITSFormat(this));
			outputFormats.add(new JSONFormat(this));
			outputFormats.add(new SVFormat(this, ",", true));
			outputFormats.add(new SVFormat(this, "\t", true));
			outputFormats.add(new TextFormat(this));
			outputFormats.add(new HTMLFormat(this));
			return;
		}

		// LIST OF FORMATS:
		// Since it is a comma separated list of output formats, a loop will parse this list comma by comma:
		String f;
		int indexSep, indexLPar, indexRPar;
		boolean hasVotableFormat = false;
		while(formats != null && formats.length() > 0) {
			// Get a format item from the list:
			indexSep = formats.indexOf(',');
			// if a comma is after a left parenthesis
			indexLPar = formats.indexOf('(');
			if (indexSep > 0 && indexLPar > 0 && indexSep > indexLPar) {
				indexRPar = formats.indexOf(')', indexLPar);
				if (indexRPar > 0)
					indexSep = formats.indexOf(',', indexRPar);
				else
					throw new TAPException("Missing right parenthesis in: \"" + formats + "\"!");
			}
			// no comma => only one format
			if (indexSep < 0) {
				f = formats;
				formats = null;
			}
			// comma at the first position => empty list item => go to the next item
			else if (indexSep == 0) {
				formats = formats.substring(1).trim();
				continue;
			}
			// else => get the first format item, and then remove it from the list for the next iteration
			else {
				f = formats.substring(0, indexSep).trim();
				formats = formats.substring(indexSep + 1).trim();
			}

			// Identify the format and append it to the output format list of the service:
			// FITS
			if (f.equalsIgnoreCase(VALUE_FITS))
				outputFormats.add(new FITSFormat(this));
			// JSON
			else if (f.equalsIgnoreCase(VALUE_JSON))
				outputFormats.add(new JSONFormat(this));
			// HTML
			else if (f.equalsIgnoreCase(VALUE_HTML))
				outputFormats.add(new HTMLFormat(this));
			// TEXT
			else if (f.equalsIgnoreCase(VALUE_TEXT))
				outputFormats.add(new TextFormat(this));
			// CSV
			else if (f.equalsIgnoreCase(VALUE_CSV))
				outputFormats.add(new SVFormat(this, ",", true));
			// TSV
			else if (f.equalsIgnoreCase(VALUE_TSV))
				outputFormats.add(new SVFormat(this, "\t", true));
			// any SV (separated value) format
			else if (f.toLowerCase().startsWith(VALUE_SV)) {
				// get the separator:
				int endSep = f.indexOf(')');
				if (VALUE_SV.length() < f.length() && f.charAt(VALUE_SV.length()) == '(' && endSep > VALUE_SV.length() + 1) {
					String separator = f.substring(VALUE_SV.length() + 1, f.length() - 1);
					// get the MIME type and its alias, if any of them is provided:
					String mimeType = null, shortMimeType = null;
					if (endSep + 1 < f.length() && f.charAt(endSep + 1) == ':') {
						int endMime = f.indexOf(':', endSep + 2);
						if (endMime < 0)
							mimeType = f.substring(endSep + 2, f.length());
						else if (endMime > 0) {
							mimeType = f.substring(endSep + 2, endMime);
							shortMimeType = f.substring(endMime + 1);
						}
					}
					// add the defined SV(...) format:
					outputFormats.add(new SVFormat(this, separator, true, mimeType, shortMimeType));
				} else
					throw new TAPException("Missing separator char/string for the SV output format: \"" + f + "\"!");
			}
			// VOTABLE
			else if (f.toLowerCase().startsWith(VALUE_VOTABLE) || f.toLowerCase().startsWith(VALUE_VOT)) {
				// Parse the format:
				VOTableFormat votFormat = parseVOTableFormat(f);

				// Add the VOTable format:
				outputFormats.add(votFormat);

				// Determine whether the MIME type is the VOTable expected one:
				if (votFormat.getShortMimeType().equals("votable") || votFormat.getMimeType().equals("votable"))
					hasVotableFormat = true;
			}
			// custom OutputFormat
			else if (isClassName(f))
				outputFormats.add(TAPConfiguration.newInstance(f, KEY_OUTPUT_FORMATS, OutputFormat.class, new Class<?>[]{ ServiceConnection.class }, new Object[]{ this }));
			// unknown format
			else
				throw new TAPException("Unknown output format: " + f);
		}

		// Add by default VOTable format if none is specified:
		if (!hasVotableFormat)
			outputFormats.add(new VOTableFormat(this));
	}

	/**
	 * <p>Parse the given VOTable format specification.</p>
	 *
	 * <p>This specification is expected to be an item of the property {@link TAPConfiguration#KEY_OUTPUT_FORMATS}.</p>
	 *
	 * @param propValue	A single VOTable format specification.
	 *
	 * @return	The corresponding configured {@link VOTableFormat} instance.
	 *
	 * @throws TAPException	If the syntax of the given specification is incorrect,
	 *                     	or if the specified VOTable version or serialization does not exist.
	 */
	private VOTableFormat parseVOTableFormat(final String propValue) throws TAPException {
		DataFormat serialization = null;
		VOTableVersion votVersion = null;
		String mimeType = null, shortMimeType = null;

		// Get the parameters, if any:
		int beginSep = propValue.indexOf('(');
		if (beginSep > 0) {
			int endSep = propValue.indexOf(')');
			if (endSep <= beginSep)
				throw new TAPException("Wrong output format specification syntax in: \"" + propValue + "\"! A VOTable parameters list must end with ')'.");
			// split the parameters:
			String[] params = propValue.substring(beginSep + 1, endSep).split(",");
			if (params.length > 2)
				throw new TAPException("Wrong number of parameters for the output format VOTable: \"" + propValue + "\"! Only two parameters may be provided: serialization and version.");
			else if (params.length >= 1) {
				// resolve the serialization format:
				params[0] = params[0].trim().toLowerCase();
				if (params[0].length() == 0 || params[0].equals("b") || params[0].equals("binary"))
					serialization = DataFormat.BINARY;
				else if (params[0].equals("b2") || params[0].equals("binary2"))
					serialization = DataFormat.BINARY2;
				else if (params[0].equals("td") || params[0].equals("tabledata"))
					serialization = DataFormat.TABLEDATA;
				else if (params[0].equals("fits"))
					serialization = DataFormat.FITS;
				else
					throw new TAPException("Unsupported VOTable serialization: \"" + params[0] + "\"! Accepted values: 'binary' (or 'b'), 'binary2' (or 'b2'), 'tabledata' (or 'td') and 'fits'.");
				// resolve the version:
				if (params.length == 2) {
					params[1] = params[1].trim();
					if (params[1].equals("1.0") || params[1].equalsIgnoreCase("v1.0"))
						votVersion = VOTableVersion.V10;
					else if (params[1].equals("1.1") || params[1].equalsIgnoreCase("v1.1"))
						votVersion = VOTableVersion.V11;
					else if (params[1].equals("1.2") || params[1].equalsIgnoreCase("v1.2"))
						votVersion = VOTableVersion.V12;
					else if (params[1].equals("1.3") || params[1].equalsIgnoreCase("v1.3"))
						votVersion = VOTableVersion.V13;
					else
						throw new TAPException("Unsupported VOTable version: \"" + params[1] + "\"! Accepted values: '1.0' (or 'v1.0'), '1.1' (or 'v1.1'), '1.2' (or 'v1.2') and '1.3' (or 'v1.3').");
				}
			}
		}

		// Get the MIME type and its alias, if any:
		beginSep = propValue.indexOf(':');
		if (beginSep > 0) {
			int endSep = propValue.indexOf(':', beginSep + 1);
			if (endSep < 0)
				endSep = propValue.length();
			// extract the MIME type, if any:
			mimeType = propValue.substring(beginSep + 1, endSep).trim();
			if (mimeType.length() == 0)
				mimeType = null;
			// extract the short MIME type, if any:
			if (endSep < propValue.length()) {
				beginSep = endSep;
				endSep = propValue.indexOf(':', beginSep + 1);
				if (endSep >= 0)
					throw new TAPException("Wrong output format specification syntax in: \"" + propValue + "\"! After a MIME type and a short MIME type, no more information is expected.");
				else
					endSep = propValue.length();
				shortMimeType = propValue.substring(beginSep + 1, endSep).trim();
				if (shortMimeType.length() == 0)
					shortMimeType = null;
			}
		}

		// Create the VOTable format:
		VOTableFormat votFormat = new VOTableFormat(this, serialization, votVersion);
		votFormat.setMimeType(mimeType, shortMimeType);

		return votFormat;
	}

	/**
	 * Initialize the default and maximum output limits.
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws TAPException	If the corresponding TAP configuration properties are wrong.
	 */
	private void initOutputLimits(final Properties tapConfig) throws TAPException {
		Object[] limit = parseLimit(getProperty(tapConfig, KEY_DEFAULT_OUTPUT_LIMIT), KEY_DEFAULT_OUTPUT_LIMIT, false);
		outputLimitTypes[0] = (LimitUnit)limit[1];	// it should be "rows" since the parameter areBytesAllowed of parseLimit =false
		setDefaultOutputLimit((Integer)limit[0]);

		limit = parseLimit(getProperty(tapConfig, KEY_MAX_OUTPUT_LIMIT), KEY_DEFAULT_OUTPUT_LIMIT, false);
		outputLimitTypes[1] = (LimitUnit)limit[1];	// it should be "rows" since the parameter areBytesAllowed of parseLimit =false
		setMaxOutputLimit((Integer)limit[0]);
	}

	/**
	 * Initialize the fetch size for the synchronous and for the asynchronous resources.
	 *
	 * @param tapConfig	The content of the TAP configuration file.
	 *
	 * @throws TAPException	If the corresponding TAP configuration properties are wrong.
	 */
	private void initFetchSize(final Properties tapConfig) throws TAPException {
		fetchSize = new int[2];

		// Set the fetch size for asynchronous queries:
		String propVal = getProperty(tapConfig, KEY_ASYNC_FETCH_SIZE);
		if (propVal == null)
			fetchSize[0] = DEFAULT_ASYNC_FETCH_SIZE;
		else {
			try {
				fetchSize[0] = Integer.parseInt(propVal);
				if (fetchSize[0] < 0)
					fetchSize[0] = 0;
			} catch(NumberFormatException nfe) {
				throw new TAPException("Integer expected for the property " + KEY_ASYNC_FETCH_SIZE + ": \"" + propVal + "\"!");
			}
		}

		// Set the fetch size for synchronous queries:
		propVal = getProperty(tapConfig, KEY_SYNC_FETCH_SIZE);
		if (propVal == null)
			fetchSize[1] = DEFAULT_SYNC_FETCH_SIZE;
		else {
			try {