Commit c6e34e6d authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Support for datatype both in ADQL and VOTable format, datatype property is now...

Support for datatype both in ADQL and VOTable format, datatype property is now editable showing a warning if the inserted value is different from the suggested value
parent 8f95d727
......@@ -22,8 +22,6 @@
*/
package it.inaf.ia2.tsm;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
......@@ -65,22 +63,6 @@ public class Column extends ChildEntity<Table> {
setStatus(Status.LOADED);
}
private void overrideDataTypeFromModels() {
Schema parentSchema = parentTable.getParent();
String type = null;
if (tapSchema.getName().equals(parentSchema.getName())) {
TableModel tableModel = tapSchema.getTapSchemaModel().getTable(parentTable.getName());
type = tableModel.get(getName()).getType();
}
if (type != null) {
String compatibleType = TypesMapping.getCompatibleADQLType(type, tapSchema.getVersion());
getProperty(DATATYPE_KEY).init(compatibleType);
}
}
public Key getForeignKey() {
if (!foreignKeySearched) { // lazy loading (but the foreignKey value can be null, so we use this boolean)
......
......@@ -37,6 +37,7 @@ public class EntityProperty<T> implements Serializable {
private ColumnModel propertyModel;
private Class<T> type;
private T defaultValue;
private T originalValue;
private T value;
private boolean changed;
......@@ -47,6 +48,7 @@ public class EntityProperty<T> implements Serializable {
public EntityProperty(ColumnModel propertyModel, T defaultValue) {
this.propertyModel = propertyModel;
this.type = propertyModel.getJavaType();
this.defaultValue = defaultValue;
this.init(defaultValue);
}
......@@ -68,6 +70,10 @@ public class EntityProperty<T> implements Serializable {
return originalValue;
}
public T getDefaultValue() {
return defaultValue;
}
public <X> X getOriginalValue(Class<X> type) {
return (X) originalValue;
}
......
......@@ -25,6 +25,7 @@ package it.inaf.ia2.tsm;
import it.inaf.ia2.tsm.datalayer.DBBroker;
import it.inaf.ia2.tsm.datalayer.DBBrokerFactory;
import it.inaf.ia2.tsm.datalayer.DBWrapper;
import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.SchemaModel;
......@@ -68,24 +69,22 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
private boolean loading;
private DBWrapper dbWrapper;
private boolean exists;
private String tapSchemaVersion;
private String tapSchemaName;
private boolean obscore;
private String obscoreVersion;
private TapSchemaSettings settings;
private DataTypeMode dataTypeMode;
private transient DBBroker sourceDBBroker;
private transient DBBroker tapSchemaDBBroker;
public final DBBroker getSourceDBBroker() {
if (sourceDBBroker == null) {
sourceDBBroker = DBBrokerFactory.getDBBroker(dbWrapper.getSourceDataSourceWrapper(), tapSchemaVersion);
sourceDBBroker = DBBrokerFactory.getDBBroker(dbWrapper.getSourceDataSourceWrapper(), dataTypeMode);
}
return sourceDBBroker;
}
public final DBBroker getTapSchemaDBBroker() {
if (tapSchemaDBBroker == null) {
tapSchemaDBBroker = DBBrokerFactory.getDBBroker(dbWrapper.getTapSchemaDataSourceWrapper(), tapSchemaVersion);
tapSchemaDBBroker = DBBrokerFactory.getDBBroker(dbWrapper.getTapSchemaDataSourceWrapper(), dataTypeMode);
}
return tapSchemaDBBroker;
}
......@@ -103,16 +102,15 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
loading = true;
this.dbWrapper = dbWrapper;
this.exists = exists;
this.tapSchemaVersion = settings.getTapSchemaVersion();
this.tapSchemaName = settings.getTapSchemaName();
this.obscore = settings.isHasObscore();
this.obscoreVersion = settings.getObscoreVersion();
this.settings = settings;
dataTypeMode = getTapSchemaModel().getDataTypeMode();
// Initializing schemas map
for (String schemaName : getSourceDBBroker().getAllSchemaNames()) {
schemas.put(schemaName, null);
}
schemas.put(tapSchemaName, null); // the TAP_SCHEMA contains itself
schemas.put(settings.getTapSchemaName(), null); // the TAP_SCHEMA contains itself
if (exists) {
loadSavedTapSchema();
......@@ -282,7 +280,7 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
}
public DBBroker getDBBroker(String schemaName) {
if (schemaName.equals(tapSchemaName)) {
if (schemaName.equals(getName())) {
return getTapSchemaDBBroker();
} else {
return getSourceDBBroker();
......@@ -293,14 +291,18 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
* The name of the TAP_SCHEMA schema.
*/
public final String getName() {
return tapSchemaName;
return settings.getTapSchemaName();
}
/**
* The version selected for this TAP_SCHEMA.
*/
public String getVersion() {
return tapSchemaVersion;
return settings.getTapSchemaVersion();
}
public DataTypeMode getDataTypeMode() {
return dataTypeMode;
}
private void loadSchemaKeysMetadata(String schemaName) throws SQLException {
......@@ -428,8 +430,8 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
}
public SchemaModel getIvoaSchemaModel() {
if (obscore) {
return SchemaModels.getIvoaSchemaModel(obscoreVersion);
if (settings.isHasObscore()) {
return SchemaModels.getIvoaSchemaModel(settings.getObscoreVersion());
}
return null;
}
......@@ -453,13 +455,13 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
if (!exists) {
SchemaModel tapSchemaModel = getTapSchemaModel();
broker.createTapSchemaStructure(tapSchemaName, tapSchemaModel);
broker.createTapSchemaStructure(getName(), tapSchemaModel);
// Adding TAP_SCHEMA into TAP_SCHEMA
addEntireSchema(tapSchemaName);
fillColumnProperties(tapSchemaModel, tapSchemaName);
addEntireSchema(getName());
fillColumnProperties(tapSchemaModel, getName());
if (obscore) {
if (settings.isHasObscore()) {
SchemaModel ivoaSchemaModel = getIvoaSchemaModel();
// ivoa schema has to be created into source database
......@@ -561,7 +563,7 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
StringBuilder sb = new StringBuilder("\n");
sb.append(String.format(">> TAP_SCHEMA %s <<\n", tapSchemaName));
sb.append(String.format(">> TAP_SCHEMA %s <<\n", getName()));
for (Schema schema : getChildren()) {
sb.append("--");
......@@ -655,8 +657,8 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
return consistencyChecks;
}
public SchemaModel getTapSchemaModel() {
return SchemaModels.getTapSchemaModel(tapSchemaVersion);
public final SchemaModel getTapSchemaModel() {
return SchemaModels.getTapSchemaModel(getVersion());
}
public final TableModel getTableModel(String tableName) {
......@@ -687,9 +689,9 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
}
public boolean isHasObscore() {
return obscore;
return settings.isHasObscore();
}
/**
* Fill descriptions of the TAP_SCHEMA schema entities for a given
* SchemaModel (TAP_SCHEMA or ivoa).
......@@ -704,15 +706,19 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
for (TableModel tableModel : schemaModel.getTables()) {
Table table = schema.getChild(tableModel.getName());
table.setValue(DESCRIPTION_KEY, tableModel.getDescription());
for (ColumnModel propertyModel : tableModel.getColumns()) {
Column column = table.getChild(propertyModel.getName());
column.setValue(DESCRIPTION_KEY, propertyModel.getDescription());
column.setValue(Column.UCD_KEY, propertyModel.getUcd());
column.setValue(Column.UNIT_KEY, propertyModel.getUnit());
column.setValue(Column.UTYPE_KEY, propertyModel.getUtype());
Object compatibleStd = useIntegerAsBool ? getIntAsBool(propertyModel.isStandard()) : propertyModel.isStandard();
Object compatiblePrincipal = useIntegerAsBool ? getIntAsBool(propertyModel.isPrincipal()) : propertyModel.isPrincipal();
for (ColumnModel columnModel : tableModel.getColumns()) {
Column column = table.getChild(columnModel.getName());
if (!columnModel.isMandatory() && column == null) {
// column could be null if it is not mandatory
continue;
}
column.setValue(DESCRIPTION_KEY, columnModel.getDescription());
column.setValue(Column.UCD_KEY, columnModel.getUcd());
column.setValue(Column.UNIT_KEY, columnModel.getUnit());
column.setValue(Column.UTYPE_KEY, columnModel.getUtype());
Object compatibleStd = useIntegerAsBool ? getIntAsBool(columnModel.isStandard()) : columnModel.isStandard();
Object compatiblePrincipal = useIntegerAsBool ? getIntAsBool(columnModel.isPrincipal()) : columnModel.isPrincipal();
column.setValue(Column.STD_KEY, compatibleStd);
column.setValue(Column.PRINCIPAL_KEY, compatiblePrincipal);
}
......
......@@ -22,6 +22,7 @@
*/
package it.inaf.ia2.tsm;
import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import java.io.Serializable;
/**
......
......@@ -29,18 +29,31 @@ package it.inaf.ia2.tsm.datalayer;
*/
public enum ADQL {
INTEGER,
VARCHAR,
CHAR,
BOOLEAN,
SMALLINT,
INTEGER,
BIGINT,
REAL,
DOUBLE,
BOOLEAN,
CHAR,
TIMESTAMP,
CLOB,
VARCHAR,
TIMESTAMP;
BLOB,
BINARY,
VARBINARY,
REGION,
POINT;
/**
* For not standard types
*/
public static String getDataType(String dataType) {
return dataType.toUpperCase();
// removing size from datatype.
return dataType.toUpperCase().replaceAll("\\(.+\\)", "");
}
public static boolean isVariable(ADQL adql) {
return adql.equals(VARCHAR) || adql.equals(VARBINARY);
}
}
......@@ -31,13 +31,13 @@ import it.inaf.ia2.tsm.datalayer.pgsql.PostgresDBBroker;
*/
public class DBBrokerFactory {
public static DBBroker getDBBroker(DataSourceWrapper dataSourceWrapper, String tapSchemaVersion) {
public static DBBroker getDBBroker(DataSourceWrapper dataSourceWrapper, DataTypeMode dataTypeMode) {
switch (dataSourceWrapper.getDatabaseType()) {
case MYSQL:
return new MySQLDBBroker(dataSourceWrapper.getDataSource(), tapSchemaVersion);
return new MySQLDBBroker(dataSourceWrapper.getDataSource(), dataTypeMode);
default:
String pgDatabase = dataSourceWrapper.getCredentials().getDatabase();
return new PostgresDBBroker(dataSourceWrapper.getDataSource(), pgDatabase, tapSchemaVersion);
return new PostgresDBBroker(dataSourceWrapper.getDataSource(), pgDatabase, dataTypeMode);
}
}
}
......@@ -62,12 +62,12 @@ public abstract class DBBrokerTemplate implements DBBroker {
protected final DataSource dataSource;
private final char escapeCharacter;
private final String tapSchemaVersion;
private final DataTypeMode dataTypeMode;
public DBBrokerTemplate(DataSource dataSource, char escapeCharacter, String tapSchemaVersion) {
public DBBrokerTemplate(DataSource dataSource, char escapeCharacter, DataTypeMode dataTypeMode) {
this.dataSource = dataSource;
this.escapeCharacter = escapeCharacter;
this.tapSchemaVersion = tapSchemaVersion;
this.dataTypeMode = dataTypeMode;
}
protected List<String> getAllItemsNames(String query) throws SQLException {
......@@ -778,20 +778,16 @@ public abstract class DBBrokerTemplate implements DBBroker {
if (tableModel != null) {
for (Map.Entry<String, Map<String, Object>> entry : metadata.entrySet()) {
String columnName = entry.getKey();
String declaredDataType = tableModel.get(columnName).getType();
String compatibleType = TypesMapping.getCompatibleADQLType(declaredDataType, tapSchemaVersion);
Integer size = (Integer) entry.getValue().get(Column.SIZE_KEY);
if (size != null) {
compatibleType += String.format("(%s)", size);
}
entry.getValue().put(Column.DATATYPE_KEY, compatibleType);
String adqlType = tableModel.get(columnName).getType();
String tapDataType = TypesMapping.getDataType(adqlType, dataTypeMode);
entry.getValue().put(Column.DATATYPE_KEY, tapDataType);
}
}
return metadata;
}
protected String getTapSchemaVersion() {
return tapSchemaVersion;
protected DataTypeMode getDataTypeMode() {
return dataTypeMode;
}
}
/*
* _____________________________________________________________________________
*
* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
* Trieste INAF - IA2 Italian Center for Astronomical Archives
* _____________________________________________________________________________
*
* Copyright (C) 2017 Istituto Nazionale di Astrofisica
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License Version 3 as published by the
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package it.inaf.ia2.tsm.datalayer;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public enum DataTypeMode {
ADQL("ADQL"),
VOTABLE("VOTable");
private final String name;
private DataTypeMode(String name) {
this.name = name;
}
public String getName() {
return name;
}
}
......@@ -27,6 +27,7 @@ import it.inaf.ia2.tsm.Key;
import it.inaf.ia2.tsm.TapSchema;
import it.inaf.ia2.tsm.datalayer.ADQL;
import it.inaf.ia2.tsm.datalayer.DBBrokerTemplate;
import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypesMapping;
......@@ -50,8 +51,8 @@ public class MySQLDBBroker extends DBBrokerTemplate {
private static final Logger LOG = LoggerFactory.getLogger(MySQLDBBroker.class);
public MySQLDBBroker(DataSource dataSource, String tapSchemaVersion) {
super(dataSource, '`', tapSchemaVersion);
public MySQLDBBroker(DataSource dataSource, DataTypeMode mode) {
super(dataSource, '`', mode);
}
@Override
......@@ -115,13 +116,12 @@ public class MySQLDBBroker extends DBBrokerTemplate {
cm.put(Column.INDEXED_KEY, indexed);
// Datatype and Size
String type = resultSet.getString("Type").toUpperCase();
String datatype = TypesMapping.getADQLTypeFromMySQLType(type, getTapSchemaVersion());
Integer size = getSize(type);
if (size != null && size > 0 && !datatype.contains("(")) {
// Adding size at the end of datatype
datatype += String.format("(%s)", size);
}
String dbType = resultSet.getString("Type").toUpperCase();
ADQL adqlType = TypesMapping.getADQLFromDBType(dbType);
String datatype = TypesMapping.getDataTypeFromMySQLType(dbType, getDataTypeMode());
Integer size = getSize(dbType);
cm.put(Column.DATATYPE_KEY, datatype);
cm.put(Column.SIZE_KEY, size);
......@@ -130,7 +130,7 @@ public class MySQLDBBroker extends DBBrokerTemplate {
if (size != null) {
arraySize = String.valueOf(size);
// variable length columns must have a "*" symbol on arraysize
if (datatype.startsWith(ADQL.VARCHAR.name())) {
if (adqlType != null && ADQL.isVariable(adqlType)) {
arraySize += "*";
}
}
......
......@@ -27,6 +27,7 @@ import it.inaf.ia2.tsm.Key;
import it.inaf.ia2.tsm.TapSchema;
import it.inaf.ia2.tsm.datalayer.ADQL;
import it.inaf.ia2.tsm.datalayer.DBBrokerTemplate;
import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypesMapping;
......@@ -53,8 +54,8 @@ public class PostgresDBBroker extends DBBrokerTemplate {
private final String pgDatabaseName;
public PostgresDBBroker(DataSource dataSource, String pgDatabaseName, String tapSchemaVersion) {
super(dataSource, '"', tapSchemaVersion);
public PostgresDBBroker(DataSource dataSource, String pgDatabaseName, DataTypeMode mode) {
super(dataSource, '"', mode);
this.pgDatabaseName = pgDatabaseName;
}
......@@ -197,28 +198,25 @@ public class PostgresDBBroker extends DBBrokerTemplate {
Integer size = null;
int arraydimension = 0;
String datatype;
String type = resultSet.getString("data_type").toUpperCase();
String dbType = resultSet.getString("data_type").toUpperCase();
boolean isArray = false;
if ("ARRAY".equals(type)) {
if ("ARRAY".equals(dbType)) {
isArray = true;
// example: integer array has data_type ARRAY and format_type integer[]
type = resultSet.getString("format_type").toUpperCase();
dbType = resultSet.getString("format_type").toUpperCase();
// example: an array defined as integer[5][5] has arraydim = 2
// unfortunately it seems there is no easy way to get also the
// numbers inside brakets, so this case will be approximated to *x*
arraydimension = resultSet.getInt("arraydim");
}
datatype = TypesMapping.getADQLTypeFromPostgresType(type, getTapSchemaVersion());
if (!isArray && (datatype.equals(ADQL.VARCHAR.name()) || datatype.equals(ADQL.CHAR.name()))) {
ADQL adqlType = TypesMapping.getADQLFromDBType(dbType);
String datatype = TypesMapping.getDataTypeFromPostgresType(dbType, getDataTypeMode());
if (!isArray && (ADQL.VARCHAR.equals(adqlType) || ADQL.CHAR.equals(adqlType))) {
size = resultSet.getInt("character_maximum_length");
}
if (size != null && size > 0 && !datatype.contains("(")) {
// Adding size at the end of datatype
datatype += String.format("(%s)", size);
}
cm.put(Column.DATATYPE_KEY, datatype);
cm.put(Column.SIZE_KEY, size);
......@@ -229,7 +227,7 @@ public class PostgresDBBroker extends DBBrokerTemplate {
} else if (size != null) {
arraySize = String.valueOf(size);
// variable length columns must have a "*" symbol on arraysize
if (datatype.startsWith(ADQL.VARCHAR.name())) {
if (adqlType != null && ADQL.isVariable(adqlType)) {
arraySize += "*";
}
}
......
/*
* _____________________________________________________________________________
*
* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
* Trieste INAF - IA2 Italian Center for Astronomical Archives
* _____________________________________________________________________________
*
* Copyright (C) 2017 Istituto Nazionale di Astrofisica
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License Version 3 as published by the
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package it.inaf.ia2.tsm.model;
import java.util.ArrayList;
import java.util.List;
import javax.xml.bind.annotation.XmlAttribute;
import javax.xml.bind.annotation.XmlElement;
import javax.xml.bind.annotation.XmlElements;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class DBTypeMapping {
private boolean inverse;
private final List<String> types;
public DBTypeMapping() {
types = new ArrayList<>();
}
@XmlAttribute(name = "inverse")
public boolean isInverse() {
return inverse;
}
public void setInverse(boolean inverse) {
this.inverse = inverse;
}
@XmlElements({
@XmlElement(name = "type")
})
public List<String> getTypes() {
return types;
}
}
/*
* _____________________________________________________________________________
*
* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
* Trieste INAF - IA2 Italian Center for Astronomical Archives
* _____________________________________________________________________________
*
* Copyright (C) 2017 Istituto Nazionale di Astrofisica
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License Version 3 as published by the
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.