adqlGrammar200.jj

/*
 * This file is part of ADQLLibrary.
 * 
 * ADQLLibrary is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * ADQLLibrary is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with ADQLLibrary.  If not, see <http://www.gnu.org/licenses/>.
 * 
 * Copyright 2019 - UDS/Centre de Données astronomiques de Strasbourg (CDS)
 */

/*
* This JavaCC file implements the BNF definition of ADQL v2.0
* (IVOA Recommendation 30 Oct 2008 - http://www.ivoa.net/Documents/cover/ADQL-20081030.html).
* 
* To generate the parser with this file use JavaCC. This .jj file has been
* successfully tested with JavaCC 6.0.
* 
* The generated parser checks the syntax of the given ADQL query and generates
* an object representation but no coherence with any database is done.
* If the syntax is not conform to the ADQL definition an error message is
* printed else it will be the message "Correct syntax".
*
*  Author:  Gr&eacute;gory Mantelet (CDS)
*  Version: 2.0 (04/2019)
*/

							/* ########### */
							/* # OPTIONS # */
							/* ########### */
options {
	STATIC = false;
	IGNORE_CASE = true;
	DEBUG_PARSER = true;
	KEEP_LINE_COLUMN = true;
	COMMON_TOKEN_ACTION = true;
}

							/* ########## */
							/* # PARSER # */
							/* ########## */
PARSER_BEGIN(ADQLParser200)

package adql.parser;

/*
 * This file is part of ADQLLibrary.
 * 
 * ADQLLibrary is free software: you can redistribute it and/or modify
 * it under the terms of the GNU Lesser General Public License as published by
 * the Free Software Foundation, either version 3 of the License, or
 * (at your option) any later version.
 * 
 * ADQLLibrary is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU Lesser General Public License for more details.
 * 
 * You should have received a copy of the GNU Lesser General Public License
 * along with ADQLLibrary.  If not, see <http://www.gnu.org/licenses/>.
 * 
 * Copyright 2019 - UDS/Centre de Données astronomiques de Strasbourg (CDS)
 */

import java.util.Stack;
import java.util.Vector;
import java.util.ArrayList;
import java.util.Collection;

import java.io.FileReader;
import java.io.IOException;

import adql.db.exception.UnresolvedIdentifiersException;

import adql.parser.IdentifierItems.IdentifierItem;

import adql.parser.ADQLQueryFactory.JoinType;

import adql.query.*;
import adql.query.from.*;
import adql.query.constraint.*;

import adql.query.operand.*;

import adql.query.operand.function.*;

import adql.query.operand.function.geometry.*;
import adql.query.operand.function.geometry.GeometryFunction.GeometryValue;

import adql.translator.PostgreSQLTranslator;
import adql.translator.TranslationException;

/**
* Parses an ADQL-2.0 query thanks to the {@link ADQLParser200#Query() Query()} function.
* 
* <p>
*   This parser is able, thanks to a {@link QueryChecker} object, to check each
*   {@link ADQLQuery} just after its generation. It could be used to check the
*   consistency between the ADQL query to parse and the "database" on which the
*   query must be executed. By default, there is no {@link QueryChecker}. Thus
*   you must extend {@link QueryChecker} to check semantically all generated
*   ADQLQuery objects.
* </p>
* 
* <p>
*   To create an object representation of the given ADQL query, this parser uses
*   a {@link ADQLQueryFactory} object. So if you want customize some object
*   (ie. CONTAINS) of this representation you just have to extend the
*   corresponding default object (ie. ContainsFunction) and to extend the
*   corresponding function of {@link ADQLQueryFactory}
*   (ie. createContains(...)).
* </p>
*
* <p>Here are the key functions to use:</p>
* <ul>
* 	<li>{@link #parseQuery(java.lang.String)} (or any of its alternatives)
* 		to parse an input ADQL query String and get its corresponding ADQL tree
*   </li>
*   <li>{@link #tryQuickFix(java.lang.String)} to try fixing the most common
* 		issues with ADQL queries (e.g. Unicode confusable characters,
* 		unescaped ADQL identifiers, SQL reserved keywords, ...)</li>
* </ul>
* 
* <p><b><u>WARNING:</u>
*   To modify this class it's strongly encouraged to modify the .jj file in the
*   section between <i>PARSER_BEGIN</i> and <i>PARSER_END</i> and to re-compile
*   it with JavaCC.
* </b></p>
*
* @see QueryChecker
* @see ADQLQueryFactory
*
* @author Gr&eacute;gory Mantelet (CDS)
* @version 2.0 (04/2019)
* @since 2.0
*/
public class ADQLParser200 implements ADQLParser {
	
	/** Tools to build the object representation of the ADQL query. */
	private ADQLQueryFactory queryFactory = new ADQLQueryFactory();
	
	/** The stack of queries (because there may be some sub-queries). */
	private Stack<ADQLQuery> stackQuery = new Stack<ADQLQuery>();
	
	/** The object representation of the ADQL query to parse.
	 * (ONLY USED DURING THE PARSING, else it is always <i>null</i>). */
	private ADQLQuery query = null;
	
	/** Checks each {@link ADQLQuery} (sub-query or not) just after their
	 * generation. */
	private QueryChecker queryChecker = null;
	
	/** The first token of a table/column name. This token is extracted by
	 * {@link #Identifier()}. */
	private Token currentIdentifierToken = null;
	
	/**
	* Builds an ADQL parser without a query to parse.
	*/
	public ADQLParser200(){
		this(new java.io.ByteArrayInputStream("".getBytes()));
		setDebug(false);
	}
	
	/**
	* Builds an ADQL parser without a query to parse but with a
	* {@link QueryChecker} and a {@link ADQLQueryFactory}.
	*
	* @param checker	The object to use to check each {@link ADQLQuery}.
	* @param factory	The object to use to build an object representation of
	*               	the given ADQL query.
	*/
	public ADQLParser200(QueryChecker checker, ADQLQueryFactory factory) {
		this();
		
		queryChecker = checker;
			
		if (factory != null)
			queryFactory = factory;
	}
	
	/**
	* Builds an ADQL parser without a query to parse but with a
	* {@link QueryChecker}.
	*
	* @param checker	The object to use to check each {@link ADQLQuery}.
	*/
	public ADQLParser200(QueryChecker checker) {
		this(checker, null);
	}
	
	/**
	* Builds an ADQL parser without a query to parse but with a
	* {@link ADQLQueryFactory}.
	*
	* @param factory	The object to use to build an object representation of
	*               	the given ADQL query.
	*/
	public ADQLParser200(ADQLQueryFactory factory) {
		this((QueryChecker)null, factory);
	}
	
	/**
	* Builds a parser with a stream containing the query to parse.
	*
	* @param stream		The stream in which the ADQL query to parse is given.
	* @param checker	The object to use to check each {@link ADQLQuery}.
	* @param factory	The object to use to build an object representation of
	*               	the given ADQL query.
	*/
	public ADQLParser200(java.io.InputStream stream, QueryChecker checker, ADQLQueryFactory factory) {
		this(stream);
		setDebug(false);
		
		setDebug(false);
		
		queryChecker = checker;
		
		if (factory != null)
			queryFactory = factory;
	}
	
	/**
	* Builds a parser with a stream containing the query to parse.
	*
	* @param stream		The stream in which the ADQL query to parse is given.
	* @param checker	The object to use to check each {@link ADQLQuery}.
	*/
	public ADQLParser200(java.io.InputStream stream, QueryChecker checker) {
		this(stream, checker, null);
	}
	
	/**
	* Builds a parser with a stream containing the query to parse.
	*
	* @param stream		The stream in which the ADQL query to parse is given.
	* @param factory	The object to use to build an object representation of
	*               	the given ADQL query.
	*/
	public ADQLParser200(java.io.InputStream stream, ADQLQueryFactory factory) {
		this(stream, (QueryChecker)null, factory);
	}
	
	/**
	* Builds a parser with a stream containing the query to parse.
	*
	* @param stream		The stream in which the ADQL query to parse is given.
	* @param encoding	The supplied encoding.
	* @param checker	The object to use to check each {@link ADQLQuery}.
	* @param factory	The object to use to build an object representation
	*               	of the given ADQL query.
	*/
	public ADQLParser200(java.io.InputStream stream, String encoding, QueryChecker checker, ADQLQueryFactory factory) {
		this(stream, encoding);
		setDebug(false);
		
		queryChecker = checker;
		
		if (factory != null)
			queryFactory = factory;
	}
	
	/**
	* Builds a parser with a stream containing the query to parse.
	*
	* @param stream		The stream in which the ADQL query to parse is given.
	* @param encoding	The supplied encoding.
	* @param checker	The object to use to check each {@link ADQLQuery}.
	*/
	public ADQLParser200(java.io.InputStream stream, String encoding, QueryChecker checker) {
		this(stream, encoding, checker, null);
	}
	
	/**
	* Builds a parser with a stream containing the query to parse.
	*
	* @param stream		The stream in which the ADQL query to parse is given.
	* @param encoding	The supplied encoding.
	* @param factory	The object to use to build an object representation
	*               	of the given ADQL query.
	*/
	public ADQLParser200(java.io.InputStream stream, String encoding, ADQLQueryFactory factory) {
		this(stream, encoding, null, factory);
	}
	
	/**
	* Builds a parser with a reader containing the query to parse.
	*
	* @param reader		The reader in which the ADQL query to parse is given.
	* @param checker	The object to use to check each {@link ADQLQuery}.
	* @param factory	The object to use to build an object representation
	*               	of the given ADQL query.
	*/
	public ADQLParser200(java.io.Reader reader, QueryChecker checker, ADQLQueryFactory factory) {
		this(reader);
		setDebug(false);
		
		setDebug(false);
		
		queryChecker = checker;

		if (factory != null)
			queryFactory = factory;
	}
	
	/**
	* Builds a parser with a reader containing the query to parse.
	*
	* @param reader		The reader in which the ADQL query to parse is given.
	* @param checker	The object to use to check each {@link ADQLQuery}.
	*/
	public ADQLParser200(java.io.Reader reader, QueryChecker checker) {
		this(reader, checker, null);
	}
	
	/**
	* Builds a parser with a reader containing the query to parse.
	*
	* @param reader		The reader in which the ADQL query to parse is given.
	* @param factory	The object to use to build an object representation
	*               	of the given ADQL query.
	*/
	public ADQLParser200(java.io.Reader reader, ADQLQueryFactory factory) {
		this(reader, null, factory);
	}
	
	/**
	* Builds a parser with another token manager.
	*
	* @param tm			The manager which associates a token to a numeric code.
	* @param checker	The object to use to check each {@link ADQLQuery }.
	* @param factory	The object to use to build an object representation
	*               	of the given ADQL query.
	*/
	public ADQLParser200(ADQLParser200TokenManager tm, QueryChecker checker, ADQLQueryFactory factory) {
		this(tm);
		setDebug(false);
		
		setDebug(false);
		
		queryChecker = checker;

		if (factory != null)
			queryFactory = factory;
	}
	
	/**
	* Builds a parser with another token manager.
	*
	* @param tm			The manager which associates a token to a numeric code.
	* @param checker	The object to use to check each {@link ADQLQuery}.
	*/
	public ADQLParser200(ADQLParser200TokenManager tm, QueryChecker checker) {
		this(tm, checker, null);
	}
	
	/**
	* Builds a parser with another token manager.
	*
	* @param tm			The manager which associates a token to a numeric code.
	* @param factory	The object to use to build an object representation of
	*               	the given ADQL query.
	*/
	public ADQLParser200(ADQLParser200TokenManager tm, ADQLQueryFactory factory) {
		this(tm, null, factory);
	}

	/* ADDITIONAL GETTERS & SETTERS */
	
	public final void setDebug(boolean debug){
		if (debug) enable_tracing();
		else       disable_tracing();
	}
	
	public final QueryChecker getQueryChecker(){
		return queryChecker;
	}
	
	public final void setQueryChecker(QueryChecker checker){
		queryChecker = checker;
	}
	
	public final ADQLQueryFactory getQueryFactory(){
		return queryFactory;
	}
	
	public final void setQueryFactory(ADQLQueryFactory factory){
		queryFactory = (factory!=null)?factory:(new ADQLQueryFactory());
	}

	/* EXCEPTION HELPER FUNCTION */
	
	private final ParseException generateParseException(Exception ex){
		if (!(ex instanceof ParseException)){
			ParseException pex = new ParseException("["+ex.getClass().getName()+"] "+ex.getMessage());
			pex.setStackTrace(ex.getStackTrace());
			return pex;
		}else
			return (ParseException)ex;
	}

	/* QUERY PARSING FUNCTIONS */

	/**
	 * Tell whether the given string is a valid ADQL regular identifier.
	 *
	 * <p>
	 * 	According to the ADQL-2.0's BNF, a regular identifier (i.e. not delimited
	 * 	; not between double quotes) must be a letter followed by a letter, digit
	 * 	or underscore. So, the following regular expression:
	 * </p>
	 * <pre>[a-zA-Z]+[a-zA-Z0-9_]*</pre>
	 *
	 * <p>This is what this function tests on the given string.</p>
	 *
	 * @param idCandidate	The string to test.
	 *
	 * @return	<code>true</code> if the given string is a valid regular
	 *        	identifier,
	 *        	<code>false</code> otherwise.
	 *
	 * @see #testRegularIdentifier(adql.parser.Token)
	 *
	 * @since 1.5
	 */
	public final boolean isRegularIdentifier(final String idCandidate) {
		return idCandidate.matches("[a-zA-Z]+[a-zA-Z0-9_]*");
	}

	/**
	 * Test the given token as an ADQL's regular identifier.
	 *
	 * <p>
	 * 	This function uses {@link #isRegularIdentifier(java.lang.String)} to
	 * 	test the given token's image. If the test fails, a
	 * 	{@link adql.parser.ParseException} is thrown.
	 * </p>
	 *
	 * @param token	The token to test.
	 *
	 * @throws ParseException	If the given token is not a valid ADQL regular
	 *                       	identifier.
	 *
	 * @see #isRegularIdentifier(java.lang.String)
	 *
	 * @since 1.5
	 */
	public final void testRegularIdentifier(final Token token) throws ParseException {
		if (!isRegularIdentifier(token.image))
			throw new ParseException("Invalid ADQL regular identifier: \""+token.image+"\"! If it aims to be a column/table name/alias, you should write it between double quotes.", new TextPosition(token));
	}
	
	/**
	* Parses the query given at the creation of this parser or in the
	* <i>ReInit</i> functions.
	*
	* @return 	The object representation of the given ADQL query.
	* 
	* @throws ParseException	If there is at least one syntactic error.
	*
	* @see ADQLParser200#Query()
	*/
	public final ADQLQuery parseQuery() throws ParseException {
		stackQuery.clear();
		query = null;
		try { 
			return Query();
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}
	}
	
	/**
	* Parses the query given in parameter.
	*
	* @param q	The ADQL query to parse.
	* 
	* @return	The object representation of the given ADQL query.
	* 
	* @throws ParseException	If there is at least one syntactic error.
	*
	* @see ADQLParser200#ReInit(java.io.InputStream)
	* @see ADQLParser200#setDebug(boolean)
	* @see ADQLParser200#Query()
	*/
	public final ADQLQuery parseQuery(String q) throws ParseException {
		stackQuery.clear();
		query = null;
		ReInit(new java.io.ByteArrayInputStream(q.getBytes()));
		try { 
			return Query();
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}
	}
	
	/**
	* Parses the query contained in the stream given in parameter.
	*
	* @param stream		The stream which contains the ADQL query to parse.
	* 
	* @return	The object representation of the given ADQL query.
	* 
	* @throws ParseException	If there is at least one syntactic error.
	*
	* @see ADQLParser200#ReInit(java.io.InputStream)
	* @see ADQLParser200#setDebug(boolean)
	* @see ADQLParser200#Query()
	*/
	public final ADQLQuery parseQuery(java.io.InputStream stream) throws ParseException {
		stackQuery.clear();
		query = null;
		ReInit(stream);
		try { 
			return Query();
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}
	}

	@Override
	public final ClauseSelect parseSelect(java.lang.String adql) throws ParseException {
	    // Set the string to parse:
		ReInit(new java.io.ByteArrayInputStream(adql.getBytes()));
		
		try {
			// Create the query:
			query = queryFactory.createQuery();

			// Parse the string as a SELECT clause:
			Select();

			// Return what's just got parsed:
			return query.getSelect();
			
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}catch(Exception ex){
			throw generateParseException(ex);
		}
	}
	
	@Override
	public final FromContent parseFrom(java.lang.String adql) throws ParseException {
		// Set the string to parse:
		ReInit(new java.io.ByteArrayInputStream(adql.getBytes()));
		
		try {
			// Create the query:
			query = queryFactory.createQuery();

			// Parse the string as a FROM clause:
			From();

			// Return what's just got parsed:
			return query.getFrom();
			
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}catch(Exception ex){
			throw generateParseException(ex);
		}
	}
	
	@Override
	public final ClauseConstraints parseWhere(java.lang.String adql) throws ParseException {
		// Set the string to parse:
		ReInit(new java.io.ByteArrayInputStream(adql.getBytes()));
		
		try {
			// Create the query:
			query = queryFactory.createQuery();

			// Parse the string as a WHERE clause:
			Where();

			// Return what's just got parsed:
			return query.getWhere();
			
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}catch(Exception ex){
			throw generateParseException(ex);
		}
	}
	
	@Override
	public final ClauseADQL<ADQLOrder> parseOrderBy(java.lang.String adql) throws ParseException {
		// Set the string to parse:
		ReInit(new java.io.ByteArrayInputStream(adql.getBytes()));
		
		try {
			// Create the query:
			query = queryFactory.createQuery();

			// Parse the string as a ORDER BY clause:
			OrderBy();

			// Return what's just got parsed:
			return query.getOrderBy();
			
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}catch(Exception ex){
			throw generateParseException(ex);
		}
	}
	
	@Override
	public final ClauseADQL<ADQLColumn> parseGroupBy(java.lang.String adql) throws ParseException {
		// Set the string to parse:
		ReInit(new java.io.ByteArrayInputStream(adql.getBytes()));
		
		try {
			// Create the query:
			query = queryFactory.createQuery();

			// Parse the string as a GROUP BY clause:
			GroupBy();

			// Return what's just got parsed:
			return query.getGroupBy();
			
		}catch(TokenMgrError tme) {
			throw new ParseException(tme);
		}catch(Exception ex){
			throw generateParseException(ex);
		}
	}

	/* CORRECTION SUGGESTION */

	/**
	 * Try fixing tokens/terms of the input ADQL query.
	 *
	 * <p>
	 * 	<b>This function does not try to fix syntactical or semantical errors.</b>
	 *  It just try to fix the most common issues in ADQL queries, such as:
	 * </p>
	 * <ul>
	 * 	<li>some Unicode characters confusable with ASCII characters (like a
	 * 		space, a dash, ...) ; this function replace them by their ASCII
	 * 		alternative,</li>
	 * 	<li>any of the following are double quoted:
	 * 		<ul>
	 * 			<li>non regular ADQL identifiers
	 * 				(e.g. <code>_RAJ2000</code>),</li>
	 * 			<li>ADQL function names used as identifiers
	 * 				(e.g. <code>distance</code>)</li>
	 * 			<li>and SQL reserved keywords
	 * 				(e.g. <code>public</code>).</li>
	 * 		</ul>
	 * 	</li>
	 * </ul>
	 *
	 * <p><i><b>Note 1:</b>
	 * 	The given stream is NOT closed by this function even if the EOF is
	 * 	reached. It is the responsibility of the caller to close it.
	 * </i></p>
	 *
	 * <p><i><b>Note 2:</b>
	 * 	This function does not use any instance variable of this parser
	 * 	(especially the InputStream or Reader provided at initialisation or
	 * 	ReInit).
	 * </i></p>
	 *
	 * @param input	Stream containing the input ADQL query to fix.
	 *
	 * @return	The suggested correction of the input ADQL query.
	 *
	 * @throws java.io.IOException	If there is any error while reading from the
	 *                            	given input stream.
	 * @throws ParseException	If any unrecognised character is encountered,
	 *                       	or if anything else prevented the tokenization
	 *                       	   of some characters/words/terms.
	 *
	 * @see #tryQuickFix(java.lang.String)
	 *
	 * @since 1.5
	 */
	public final String tryQuickFix(final java.io.InputStream input) throws java.io.IOException, ParseException {
		// Fetch everything into a single string:
		StringBuffer buf = new StringBuffer();
		byte[] cBuf = new byte[1024];
		int nbChar;
		while((nbChar = input.read(cBuf)) > -1){
			buf.append(new String(cBuf, 0, nbChar));
		}
		
		// Convert the buffer into a String and now try to fix it:
		return tryQuickFix(buf.toString());
	}

	/**
	 * Try fixing tokens/terms of the given ADQL query.
	 *
	 * <p>
	 * 	<b>This function does not try to fix syntactical or semantical errors.</b>
	 *  It just try to fix the most common issues in ADQL queries, such as:
	 * </p>
	 * <ul>
	 * 	<li>some Unicode characters confusable with ASCII characters (like a
	 * 		space, a dash, ...) ; this function replace them by their ASCII
	 * 		alternative,</li>
	 * 	<li>any of the following are double quoted:
	 * 		<ul>
	 * 			<li>non regular ADQL identifiers
	 * 				(e.g. <code>_RAJ2000</code>),</li>
	 * 			<li>ADQL function names used as identifiers
	 * 				(e.g. <code>distance</code>)</li>
	 * 			<li>and SQL reserved keywords
	 * 				(e.g. <code>public</code>).</li>
	 * 		</ul>
	 * 	</li>
	 * </ul>
	 *
	 * <p><i><b>Note:</b>
	 * 	This function does not use any instance variable of this parser
	 * 	(especially the InputStream or Reader provided at initialisation or
	 * 	ReInit).
	 * </i></p>
	 *
	 * @param adqlQuery	The input ADQL query to fix.
	 *
	 * @return	The suggested correction of the given ADQL query.
	 *
	 * @throws ParseException	If any unrecognised character is encountered,
	 *                       	or if anything else prevented the tokenization
	 *                       	   of some characters/words/terms.
	 *
	 * @since 1.5
	 */
	public String tryQuickFix(String adqlQuery) throws ParseException {
		StringBuffer suggestedQuery = new StringBuffer();

		// 1. Replace all Unicode confusable characters:
		adqlQuery = replaceUnicodeConfusables(adqlQuery);

		/* 1.bis. Normalise new lines and tabulations
		 *        (to simplify the column counting): */
		adqlQuery = adqlQuery.replaceAll("(\r\n|\r|\n)", System.getProperty("line.separator")).replaceAll("\t", "    ");

		// 2. Analyse the query token by token:
		ADQLParser200TokenManager parser = new ADQLParser200TokenManager(new SimpleCharStream(new java.io.ByteArrayInputStream(adqlQuery.getBytes())));
		
		final String[] lines = adqlQuery.split(System.getProperty("line.separator"));

		try{
			String suggestedToken;
			int lastLine = 1, lastCol = 1;

			Token token = null, nextToken = parser.getNextToken();
			// for all tokens until the EOF or EOQ:
			do{
				// get the next token:
				token = nextToken;
				nextToken = (isEnd(token) ? null : parser.getNextToken());

				// 3. Double quote any suspect token:
				if (mustEscape(token, nextToken)){
					suggestedToken = "\"" + token.image + "\"";
				}else
					suggestedToken = token.image;

				/* 4. Append all space characters (and comments) before the
				 *    token: */
				/* same line, just get the space characters between the last
				 * token and the one to append: */
				if (lastLine == token.beginLine){
					suggestedQuery.append(lines[lastLine - 1].substring(lastCol - 1, token.beginColumn - (isEnd(token) ? 0 : 1)));
					lastCol = token.endColumn + 1;
				}
				// not the same line...
				else{
				    /* append all remaining space characters until the position
				     * of the token to append: */
					do{
						suggestedQuery.append(lines[lastLine - 1].substring(lastCol - 1)).append('\n');
						lastLine++;
						lastCol = 1;
					}while(lastLine < token.beginLine);
					/* if there are still space characters before the token,
					 * append them as well: */
					if (lastCol < token.beginColumn)
						suggestedQuery.append(lines[lastLine - 1].substring(lastCol - 1, token.beginColumn - 1));
					// finally, set the correct column position:
					lastCol = token.endColumn + 1;
				}

				// 5. Append the suggested token:
				suggestedQuery.append(suggestedToken);

			}while(!isEnd(token));

		}catch(TokenMgrError err){
		    // wrap such errors and propagate them:
			throw new ParseException(err);
		}

		return suggestedQuery.toString();
	}

	/**
	 * All of the most common Unicode confusable characters and their
	 * ASCII/UTF-8 alternative.
	 *
	 * <p>
	 * 	Keys of this map represent the ASCII character while the values are the
	 * 	regular expression for all possible Unicode alternatives.
	 * </p>
	 *
	 * <p><i><b>Note:</b>
	 * 	All of them have been listed using
	 * 	<a href="https://unicode.org/cldr/utility/confusables.jsp">Unicode Utilities: Confusables</a>.
	 * </i></p>
	 *
	 * @since 1.5
	 */
	protected final static java.util.Map<String, String> REGEX_UNICODE_CONFUSABLES = new java.util.HashMap<String, String>(10);
	/** Regular expression matching all Unicode alternatives for <code>-</code>.
	 * @since 1.5 */
	protected final static String REGEX_DASH         = "[\u002D\u02D7\u06D4\u2010\u2011\u2012\u2013\u2043\u2212\u2796\u2CBA\uFE58\u2014\u2015\u207B\u208B\u0096\u058A\uFE63\uFF0D]";
	/** Regular expression matching all Unicode alternatives for <code>_</code>.
	 * @since 1.5 */
	protected final static String REGEX_UNDERSCORE   = "[\u005F\u07FA\uFE4D\uFE4E\uFE4F]";
	/** Regular expression matching all Unicode alternatives for <code>'</code>.
	 * @since 1.5 */
	protected final static String REGEX_QUOTE        = "[\u0027\u0060\u00B4\u02B9\u02BB\u02BC\u02BD\u02BE\u02C8\u02CA\u02CB\u02F4\u0374\u0384\u055A\u055D\u05D9\u05F3\u07F4\u07F5\u144A\u16CC\u1FBD\u1FBF\u1FEF\u1FFD\u1FFE\u2018\u2019\u201B\u2032\u2035\uA78C\uFF07\uFF40]";
	/** Regular expression matching all Unicode alternatives for <code>"</code>.
	 * @since 1.5 */
	protected final static String REGEX_DOUBLE_QUOTE = "[\u02BA\u02DD\u02EE\u02F6\u05F2\u05F4\u1CD3\u201C\u201D\u201F\u2033\u2036\u3003\uFF02]";
	/** Regular expression matching all Unicode alternatives for <code>.</code>.
	 * @since 1.5 */
	protected final static String REGEX_STOP         = "[\u002E\u0660\u06F0\u0701\u0702\u2024\uA4F8\uA60E]";
	/** Regular expression matching all Unicode alternatives for <code>+</code>.
	 * @since 1.5 */
	protected final static String REGEX_PLUS         = "[\u002B\u16ED\u2795]";
	/** Regular expression matching all Unicode alternatives for <code> </code>.
	 * @since 1.5 */
	protected final static String REGEX_SPACE        = "[\u0020\u00A0\u1680\u2000\u2001\u2002\u2003\u2004\u2005\u2006\u2007\u2008\u2009\u200A\u202F\u205F]";
	/** Regular expression matching all Unicode alternatives for <code>&lt;</code>.
	 * @since 1.5 */
	protected final static String REGEX_LESS_THAN    = "[\u003C\u02C2\u1438\u16B2\u2039\u276E]";
	/** Regular expression matching all Unicode alternatives for <code>&gt;</code>.
	 * @since 1.5 */
	protected final static String REGEX_GREATER_THAN = "[\u003E\u02C3\u1433\u203A\u276F]";
	/** Regular expression matching all Unicode alternatives for <code>=</code>.
	 * @since 1.5 */
	protected final static String REGEX_EQUAL        = "[\u003D\u1400\u2E40\u30A0\uA4FF]";
	static {
		REGEX_UNICODE_CONFUSABLES.put("-", REGEX_DASH);
		REGEX_UNICODE_CONFUSABLES.put("_", REGEX_UNDERSCORE);
		REGEX_UNICODE_CONFUSABLES.put("'", REGEX_QUOTE);
		REGEX_UNICODE_CONFUSABLES.put("\"", REGEX_DOUBLE_QUOTE);
		REGEX_UNICODE_CONFUSABLES.put(".", REGEX_STOP);
		REGEX_UNICODE_CONFUSABLES.put("+", REGEX_PLUS);
		REGEX_UNICODE_CONFUSABLES.put(" ", REGEX_SPACE);
		REGEX_UNICODE_CONFUSABLES.put("<", REGEX_LESS_THAN);
		REGEX_UNICODE_CONFUSABLES.put(">", REGEX_GREATER_THAN);
		REGEX_UNICODE_CONFUSABLES.put("=", REGEX_EQUAL);
	}

	/**
	 * Replace all Unicode characters that can be confused with other ASCI/UTF-8
	 * characters (e.g. different spaces, dashes, ...) in their ASCII version.
	 *
	 * @param adqlQuery	The ADQL query string in which Unicode confusable
	 *                 	characters must be replaced.
	 *
	 * @return	The same query without the most common Unicode confusable
	 *        	characters.
	 *
	 * @since 1.5
	 */
	protected String replaceUnicodeConfusables(final String adqlQuery){
		String newAdqlQuery = adqlQuery;
		for(java.util.Map.Entry<String, String> confusable : REGEX_UNICODE_CONFUSABLES.entrySet())
			newAdqlQuery = newAdqlQuery.replaceAll(confusable.getValue(), confusable.getKey());
		return newAdqlQuery;
	}

	/**
	 * Tell whether the given token represents the end of an ADQL query.
	 *
	 * @param token	Token to analyze.
	 *
	 * @return	<code>true</code> if the given token represents a query end,
	 *        	<code>false</code> otherwise.
	 *
	 * @since 1.5
	 */
	protected boolean isEnd(final Token token){
		return token.kind == ADQLParser200Constants.EOF || token.kind == ADQLParser200Constants.EOQ;
	}

	/**
	 * Tell whether the given token must be double quoted.
	 *
	 * <p>
	 * 	This function considers all the following as terms to double quote:
	 * </p>
	 * <ul>
	 * 	<li>SQL reserved keywords</li>,
	 * 	<li>unrecognised regular identifiers (e.g. neither a delimited nor a
	 * 		valid ADQL regular identifier)</li>
	 * 	<li>and ADQL function name without a parameters list.</li>
	 * </ul>
	 *
	 * @param token		The token to analyze.
	 * @param nextToken	The following token. (useful to detect the start of a
	 *                 	function's parameters list)
	 *
	 * @return	<code>true</code> if the given token must be double quoted,
	 *        	<code>false</code> to keep it as provided.
	 *
	 * @since 1.5
	 */
	protected boolean mustEscape(final Token token, final Token nextToken){
		switch(token.kind){
			case ADQLParser200Constants.SQL_RESERVED_WORD:
				return true;
			case ADQLParser200Constants.REGULAR_IDENTIFIER_CANDIDATE:
				return !isRegularIdentifier(token.image);
			default:
				return token.isFunctionName && (nextToken == null || nextToken.kind != ADQLParser200Constants.LEFT_PAR);
		}
	}
}

PARSER_END(ADQLParser200)

				/* ################################### */
				/* # CUSTOMIZATION OF TOKEN CREATION # */
				/* ################################### */

TOKEN_MGR_DECLS: {
	protected void CommonTokenAction(final Token t) {
		t.adqlVersion = ADQLParserFactory.ADQLVersion.V2_0;
	} 
}

							/* ########### */
							/* # GRAMMAR # */
							/* ########### */
/* ******************** */
/* Characters to ignore */
/* ******************** */
SKIP : { < " " | "\t" | "\n" | "\r" | "\r\n" > }

/* ************************************************************************** */
/* Reserved SQL words                                                         */
/*                                                                            */
/* NOTE:                                                                      */
/*   This list is the one provided by the ADQL-2.0 standard after removal of  */
/*   all ADQL used words (e.g. SELECT, AS, LIKE, AVG, ABS, COS, POINT).       */
/*   (see ParseException.initialise(Token, int[][], String[]) for more        */
/*   details)                                                                 */
/* ************************************************************************** */

TOKEN : {
	< SQL_RESERVED_WORD: ("ABSOLUTE"|"ACTION"|"ADD"|"ALLOCATE"|"ALTER"|"ANY"|"ARE"|"ASSERTION"|"AT"|"AUTHORIZATION"|"BEGIN"|"BIT"|"BIT_LENGTH"|"BOTH"|"CASCADE"|"CASCADED"|"CASE"|"CAST"|"CATALOG"|"CHAR"|"CHARACTER"|"CHAR_LENGTH"|"CHARACTER_LENGTH"|"CHECK"|"CLOSE"|"COALESCE"|"COLLATE"|"COLLATION"|"COLUMN"|"COMMIT"|"CONNECT"|"CONNECTION"|"CONSTRAINT"|"CONSTRAINTS"|"CONTINUE"|"CONVERT"|"CORRESPONDING"|"CREATE"|"CURRENT"|"CURRENT_DATE"|"CURRENT_TIME"|"CURRENT_TIMESTAMP"|"CURRENT_USER"|"CURSOR"|"DATE"|"DAY"|"DEALLOCATE"|"DECIMAL"|"DECLARE"|"DEFAULT"|"DEFERRABLE"|"DEFERRED"|"DELETE"|"DESCRIBE"|"DESCRIPTOR"|"DIAGNOSTICS"|"DISCONNECT"|"DOMAIN"|"DOUBLE"|"DROP"|"ELSE"|"END"|"END-EXEC"|"ESCAPE"|"EXCEPT"|"EXCEPTION"|"EXEC"|"EXECUTE"|"EXTERNAL"|"EXTRACT"|"FALSE"|"FETCH"|"FIRST"|"FLOAT"|"FOR"|"FOREIGN"|"FOUND"|"GET"|"GLOBAL"|"GO"|"GOTO"|"GRANT"|"HOUR"|"IDENTITY"|"IMMEDIATE"|"INDICATOR"|"INITIALLY"|"INPUT"|"INSENSITIVE"|"INSERT"|"INT"|"INTEGER"|"INTERSECT"|"INTERVAL"|"INTO"|"ISOLATION"|"KEY"|"LANGUAGE"|"LAST"|"LEADING"|"LEVEL"|"LOCAL"|"LOWER"|"MATCH"|"MINUTE"|"MODULE"|"MONTH"|"NAMES"|"NATIONAL"|"NCHAR"|"NEXT"|"NO"|"NULLIF"|"NUMERIC"|"OCTET_LENGTH"|"OF"|"ONLY"|"OPEN"|"OPTION"|"OUTPUT"|"OVERLAPS"|"PAD"|"PARTIAL"|"POSITION"|"PRECISION"|"PREPARE"|"PRESERVE"|"PRIMARY"|"PRIOR"|"PRIVILEGES"|"PROCEDURE"|"PUBLIC"|"READ"|"REAL"|"REFERENCES"|"RELATIVE"|"RESTRICT"|"REVOKE"|"ROLLBACK"|"ROWS"|"SCHEMA"|"SCROLL"|"SECOND"|"SECTION"|"SESSION"|"SESSION_USER"|"SET"|"SIZE"|"SMALLINT"|"SOME"|"SPACE"|"SQL"|"SQLCODE"|"SQLERROR"|"SQLSTATE"|"SUBSTRING"|"SYSTEM_USER"|"TABLE"|"TEMPORARY"|"THEN"|"TIME"|"TIMESTAMP"|"TIMEZONE_HOUR"|"TIMEZONE_MINUTE"|"TO"|"TRAILING"|"TRANSACTION"|"TRANSLATE"|"TRANSLATION"|"TRIM"|"TRUE"|"UNION"|"UNIQUE"|"UNKNOWN"|"UPDATE"|"UPPER"|"USAGE"|"USER"|"VALUE"|"VALUES"|"VARCHAR"|"VARYING"|"VIEW"|"WHEN"|"WHENEVER"|"WITH"|"WORK"|"WRITE"|"YEAR"|"ZONE") >
	{ matchedToken.sqlReserved = true; }
}

/* *********** */
/* Punctuation */
/* *********** */
TOKEN : {
	< LEFT_PAR: "(" >
|	< RIGHT_PAR: ")" > 
|	< DOT: "." >
|	< COMMA: "," >
|	< EOQ: ";">
|	< CONCAT: "||" >
}

/* ******************** */
/* Arithmetic operators */
/* ******************** */
TOKEN : {
	< PLUS: "+" >
|	< MINUS: "-" >
|	< ASTERISK: "*" >
|	< DIVIDE: "/" >
}

/* ******************** */
/* Comparison operators */
/* ******************** */
TOKEN : {
	< EQUAL: "=" >
|	< NOT_EQUAL: "<>" | "!=" >