Commit a29cf9e5 authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Started implementation of obscore creation

parent 9d856c18
......@@ -46,7 +46,7 @@ public class EntityProperty<T> implements Serializable {
public EntityProperty(PropertyModel propertyModel, T defaultValue) {
this.propertyModel = propertyModel;
this.type = propertyModel.getType();
this.type = propertyModel.getJavaType();
this.init(defaultValue);
}
......
......@@ -27,8 +27,8 @@ import it.inaf.ia2.tsm.datalayer.DBBrokerFactory;
import it.inaf.ia2.tsm.datalayer.DBWrapper;
import it.inaf.ia2.tsm.model.PropertyModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TapSchemaModel;
import it.inaf.ia2.tsm.model.TapSchemaModels;
import it.inaf.ia2.tsm.model.SchemaModel;
import it.inaf.ia2.tsm.model.SchemaModels;
import java.io.Serializable;
import java.sql.SQLException;
import java.util.ArrayList;
......@@ -67,10 +67,12 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
private final ConsistencyChecks consistencyChecks;
private boolean loading;
private String version;
private DBWrapper dbWrapper;
private String tapSchemaName;
private boolean exists;
private String tapSchemaVersion;
private String tapSchemaName;
private boolean obscore;
private String obscoreVersion;
private transient DBBroker sourceDBBroker;
private transient DBBroker tapSchemaDBBroker;
......@@ -96,14 +98,16 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
consistencyChecks = new ConsistencyChecks();
}
public TapSchema(String version, DBWrapper dbWrapper, String tapSchemaName, boolean exists) throws SQLException {
public TapSchema(DBWrapper dbWrapper, TapSchemaSettings settings, boolean exists) throws SQLException {
this();
loading = true;
this.version = version;
this.dbWrapper = dbWrapper;
this.tapSchemaName = tapSchemaName;
this.exists = exists;
this.tapSchemaVersion = settings.getTapSchemaVersion();
this.tapSchemaName = settings.getTapSchemaName();
this.obscore = settings.isHasObscore();
this.obscoreVersion = settings.getObscoreVersion();
// Initializing schemas map
for (String schemaName : getSourceDBBroker().getAllSchemaNames()) {
......@@ -278,10 +282,6 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
}
}
public void addFictitiousKey(String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) {
// TODO
}
public DBBroker getDBBroker(String schemaName) {
if (schemaName.equals(tapSchemaName)) {
return getTapSchemaDBBroker();
......@@ -301,7 +301,7 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
* The version selected for this TAP_SCHEMA.
*/
public String getVersion() {
return version;
return tapSchemaVersion;
}
private void loadSchemaKeysMetadata(String schemaName) throws SQLException {
......@@ -428,6 +428,23 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
schemas.put(schemaName, null);
}
private SchemaModel getIvoaSchemaModel() {
if (obscore) {
return SchemaModels.getIvoaSchemaModel(obscoreVersion);
}
return null;
}
private void addEntireSchema(String schemaName) throws SQLException {
Schema schema = addChild(schemaName);
for (String tableName : schema.getAddableChildrenNames()) {
Table table = schema.addChild(tableName);
for (String columnName : table.getAddableChildrenNames()) {
table.addChild(columnName);
}
}
}
/**
* Save or update the TAP_SCHEMA changes into the database.
*/
......@@ -436,18 +453,22 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
DBBroker broker = getTapSchemaDBBroker();
if (!exists) {
broker.createTapSchemaStructure(getName(), getTapSchemaModel());
SchemaModel tapSchemaModel = getTapSchemaModel();
broker.createTapSchemaStructure(tapSchemaName, tapSchemaModel);
// Adding TAP_SCHEMA into TAP_SCHEMA
Schema tapSchemaSchema = addChild(tapSchemaName);
for (String tableName : tapSchemaSchema.getAddableChildrenNames()) {
Table table = tapSchemaSchema.addChild(tableName);
for (String columnName : table.getAddableChildrenNames()) {
table.addChild(columnName);
}
}
addEntireSchema(tapSchemaName);
fillColumnDescriptionsAndStd(tapSchemaModel, tapSchemaName);
if (obscore) {
SchemaModel ivoaSchemaModel = getIvoaSchemaModel();
fillTapSchemaDescriptions();
broker.createIvoaSchemaStructure(ivoaSchemaModel);
// Add ivoa schema into TAP_SCHEMA
addEntireSchema(ivoaSchemaModel.getName());
fillColumnDescriptionsAndStd(ivoaSchemaModel);
}
}
fillKeyIds();
......@@ -493,20 +514,6 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
}
}
// public void addFictitiousKey(Table fromTable, String[] fromColumns, Table targetTable, String[] targetColumns) {
// Key key = new Key(dbWrapper, this, fromTable.getCompleteName(), targetTable.getCompleteName());
// key.setId((getMaxKeyId() + 1) + "");
//
// for (int i = 0; i < fromColumns.length; i++) {
// key.addKeyColumn(fromColumns[i], targetColumns[i]);
// }
//
// fromTable.addFromKey(key);
// targetTable.addTargetKey(key);
//
// allKeys.add(key);
// checkKeys();
// }
/**
* Set keys visibility based on other entities visibility (a key is visible
* if all schemas, tables and columns involved have
......@@ -646,8 +653,8 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
return consistencyChecks;
}
public TapSchemaModel getTapSchemaModel() {
return TapSchemaModels.getTapSchemaModel(version);
public SchemaModel getTapSchemaModel() {
return SchemaModels.getTapSchemaModel(tapSchemaVersion);
}
public final TableModel getTableModel(String tableName) {
......@@ -671,23 +678,26 @@ public class TapSchema implements EntitiesContainer<Schema>, Serializable {
}
/**
* Fill descriptions of the TAP_SCHEMA schema entities.
* Fill descriptions of the TAP_SCHEMA schema entities for a given
* SchemaModel (TAP_SCHEMA or ivoa).
*/
private void fillTapSchemaDescriptions() {
TapSchemaModel tapSchemaModel = getTapSchemaModel();
boolean hasStd = tapSchemaModel.get(TapSchema.COLUMNS_TABLE).get(STD_KEY) != null;
Schema tapSchema = getChild(tapSchemaName);
tapSchema.setValue(DESCRIPTION_KEY, tapSchemaModel.getDescription());
private void fillColumnDescriptionsAndStd(SchemaModel schemaModel, String schemaName) {
Schema schema = getChild(schemaName);
schema.setValue(DESCRIPTION_KEY, schemaModel.getDescription());
for (TableModel tableModel : getTapSchemaModel().getTables().values()) {
Table table = tapSchema.getChild(tableModel.getName());
tapSchema.setValue(DESCRIPTION_KEY, tableModel.getDescription());
Table table = schema.getChild(tableModel.getName());
schema.setValue(DESCRIPTION_KEY, tableModel.getDescription());
for (PropertyModel propertyModel : tableModel.getProperties().values()) {
Column column = table.getChild(propertyModel.getName());
tapSchema.setValue(DESCRIPTION_KEY, propertyModel.getDescription());
if (hasStd) {
schema.setValue(DESCRIPTION_KEY, propertyModel.getDescription());
if (propertyModel.isStandard()) {
column.setValue(STD_KEY, 1);
}
}
}
}
private void fillColumnDescriptionsAndStd(SchemaModel schemaModel) {
fillColumnDescriptionsAndStd(schemaModel, schemaModel.getName());
}
}
......@@ -24,6 +24,7 @@ package it.inaf.ia2.tsm;
import it.inaf.ia2.tsm.model.PropertyModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.HashMap;
......@@ -68,12 +69,16 @@ public abstract class TapSchemaEntity implements Serializable {
Object defaultValue = null;
if (propModel.getLoaderKey() != null) {
defaultValue = metadata.get(propModel.getLoaderKey());
if (defaultValue != null && propModel.getType() != defaultValue.getClass()) {
if (defaultValue != null) {
// Special case for boolean to integer conversion
if (defaultValue.getClass() == Boolean.class && propModel.getType() == Integer.class) {
defaultValue = ((Boolean) defaultValue) ? 1 : 0;
} else {
throw new UnsupportedOperationException("Unable to convert " + defaultValue.getClass().getCanonicalName() + " into " + propModel.getType());
Class javaType = propModel.getJavaType();
if (javaType != defaultValue.getClass()) {
if (defaultValue.getClass() == Boolean.class && javaType == Integer.class) {
defaultValue = ((Boolean) defaultValue) ? 1 : 0;
} else {
throw new UnsupportedOperationException("Unable to convert " + defaultValue.getClass().getCanonicalName() + " into " + propModel.getType());
}
}
}
}
......
......@@ -20,33 +20,52 @@
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package it.inaf.ia2.tsm.model;
package it.inaf.ia2.tsm;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.Properties;
import java.io.Serializable;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class Tasman {
public static final String[] XML_MODEL_FILES;
static {
try (InputStream in = Tasman.class.getClassLoader().getResourceAsStream("core.properties")) {
Properties props = new Properties();
props.load(in);
String[] models = props.getProperty("models").split(",");
XML_MODEL_FILES = new String[models.length];
for (int i = 0; i < models.length; i++) {
String suffix = models[i];
XML_MODEL_FILES[i] = "tap_schema" + File.separator + "tap_schema-" + suffix + ".xml";
}
} catch (IOException e) {
throw new ExceptionInInitializerError(e);
}
public class TapSchemaSettings implements Serializable {
private static final long serialVersionUID = -4152480696306984772L;
private String tapSchemaVersion;
private String tapSchemaName;
private boolean hasObscore;
private String obscoreVersion;
public String getTapSchemaVersion() {
return tapSchemaVersion;
}
public void setTapSchemaVersion(String tapSchemaVersion) {
this.tapSchemaVersion = tapSchemaVersion;
}
public String getTapSchemaName() {
return tapSchemaName;
}
public void setTapSchemaName(String tapSchemaName) {
this.tapSchemaName = tapSchemaName;
}
public boolean isHasObscore() {
return hasObscore;
}
public void setHasObscore(boolean hasObscore) {
this.hasObscore = hasObscore;
}
public String getObscoreVersion() {
return obscoreVersion;
}
public void setObscoreVersion(String obscoreVersion) {
this.obscoreVersion = obscoreVersion;
}
}
......@@ -24,22 +24,23 @@ package it.inaf.ia2.tsm.datalayer;
/**
* ADQL data type constants.
*
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class ADQL {
public enum ADQL {
private static final String ADQL_PREFIX = "adql:";
public static final String INTEGER = ADQL_PREFIX + "INTEGER";
public static final String SMALLINT = ADQL_PREFIX + "SMALLINT";
public static final String BIGINT = ADQL_PREFIX + "BIGINT";
public static final String REAL = ADQL_PREFIX + "REAL";
public static final String CHAR = ADQL_PREFIX + "CHAR";
public static final String VARCHAR = ADQL_PREFIX + "VARCHAR";
public static final String TIMESTAMP = ADQL_PREFIX + "TIMESTAMP";
INTEGER,
SMALLINT,
BIGINT,
REAL,
DOUBLE,
BOOLEAN,
CHAR,
CLOB,
VARCHAR,
TIMESTAMP;
public static String getDataType(String dataType) {
return ADQL_PREFIX + dataType.toUpperCase();
return dataType.toUpperCase();
}
}
......@@ -26,7 +26,7 @@ import it.inaf.ia2.tsm.Key;
import it.inaf.ia2.tsm.TapSchema;
import it.inaf.ia2.tsm.TapSchemaEntity;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TapSchemaModel;
import it.inaf.ia2.tsm.model.SchemaModel;
import java.sql.Connection;
import java.sql.SQLException;
import java.util.List;
......@@ -64,7 +64,9 @@ public interface DBBroker {
void updateItem(String tapSchemaName, TapSchemaEntity entity, Connection conn, String whereCondition, Object... whereParams) throws SQLException;
void createTapSchemaStructure(String tapSchemaName, TapSchemaModel tapSchemaModel) throws SQLException;
void createTapSchemaStructure(String tapSchemaName, SchemaModel tapSchemaModel) throws SQLException;
void createIvoaSchemaStructure(SchemaModel ivoaSchemaModel) throws SQLException;
void save(TapSchema tapSchema) throws SQLException;
}
......@@ -34,8 +34,9 @@ import it.inaf.ia2.tsm.TapSchemaEntity;
import it.inaf.ia2.tsm.UpdateOperations;
import it.inaf.ia2.tsm.model.PropertyModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TapSchemaModel;
import it.inaf.ia2.tsm.model.TapSchemaModels;
import it.inaf.ia2.tsm.model.SchemaModel;
import it.inaf.ia2.tsm.model.SchemaModels;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
......@@ -45,7 +46,6 @@ import java.sql.Types;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import javax.sql.DataSource;
......@@ -106,6 +106,10 @@ public abstract class DBBrokerTemplate implements DBBroker {
return String.format("%s%s%s", escapeCharacter, name, escapeCharacter);
}
protected void appendSize(StringBuilder sb, int size) {
sb.append(String.format("(%s)", size));
}
protected abstract void createTable(String tapSchemaName, TableModel tableModel, Connection conn) throws SQLException;
protected abstract String getAddPrimaryKeyQuery(String tapSchemaName, String tableName, String[] keyColumns);
......@@ -130,7 +134,7 @@ public abstract class DBBrokerTemplate implements DBBroker {
}
@Override
public void createTapSchemaStructure(String tapSchemaName, TapSchemaModel tapSchemaModel) throws SQLException {
public void createTapSchemaStructure(String tapSchemaName, SchemaModel tapSchemaModel) throws SQLException {
try (Connection conn = dataSource.getConnection()) {
......@@ -164,6 +168,16 @@ public abstract class DBBrokerTemplate implements DBBroker {
}
}
@Override
public void createIvoaSchemaStructure(SchemaModel ivoaSchemaModel) throws SQLException {
try (Connection conn = dataSource.getConnection()) {
execute(getCreateDatabaseQuery(ivoaSchemaModel.getName()), conn);
for (TableModel tableModel : ivoaSchemaModel.getTables().values()) {
createTable(ivoaSchemaModel.getName(), tableModel, conn);
}
}
}
@Override
public void save(TapSchema tapSchema) throws SQLException {
......@@ -416,7 +430,7 @@ public abstract class DBBrokerTemplate implements DBBroker {
Map<String, Object> item = new HashMap<>();
for (PropertyModel pm : tableModel.getProperties().values()) {
Object value = TSMUtil.getObject(rs, pm.getName(), pm.getType());
Object value = TSMUtil.getObject(rs, pm.getName(), pm.getJavaType());
item.put(pm.getName(), value);
}
......@@ -487,9 +501,10 @@ public abstract class DBBrokerTemplate implements DBBroker {
int i = 1;
for (String key : tapSchemaItem.getPropertiesKeys()) {
Class type = tapSchemaItem.getTableModel().get(key).getType();
Object value = tapSchemaItem.getValue(key, type);
statement.setObject(i, value, getSQLType(type));
String adqlType = tapSchemaItem.getTableModel().get(key).getType();
Class javaType = TypesMapping.getClassFromAdqlType(adqlType);
Object value = tapSchemaItem.getValue(key, javaType);
statement.setObject(i, value, getSQLType(javaType));
i++;
if (values != null) {
......@@ -663,10 +678,7 @@ public abstract class DBBrokerTemplate implements DBBroker {
keyColumnsColumns = getColumns(tapSchemaName, TapSchema.KEY_COLUMNS_TABLE, connection);
}
Iterator<TapSchemaModel> ite = TapSchemaModels.getIterator();
while (ite.hasNext()) {
TapSchemaModel tapSchemaModel = ite.next();
for (SchemaModel tapSchemaModel : SchemaModels.getTapSchemaModels()) {
if (match(tapSchemaModel.get(TapSchema.SCHEMAS_TABLE), schemasColumns)
&& match(tapSchemaModel.get(TapSchema.TABLES_TABLE), tablesColumns)
&& match(tapSchemaModel.get(TapSchema.COLUMNS_TABLE), columnsColumns)
......
......@@ -29,6 +29,7 @@ import it.inaf.ia2.tsm.datalayer.ADQL;
import it.inaf.ia2.tsm.datalayer.DBBrokerTemplate;
import it.inaf.ia2.tsm.model.PropertyModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
......@@ -68,6 +69,17 @@ public class MySQLDBBroker extends DBBrokerTemplate {
return String.format("SHOW COLUMNS FROM %s.%s", escape(schemaName), escape(tableName));
}
private Integer getSize(String typeWithSize) {
int beginIndex = typeWithSize.indexOf('(');
if (beginIndex != -1) {
int endIndex = typeWithSize.indexOf(')');
if (endIndex != -1) {
Integer.parseInt(typeWithSize.substring(beginIndex + 1, endIndex));
}
}
return null;
}
@Override
public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName) throws SQLException {
......@@ -100,32 +112,8 @@ public class MySQLDBBroker extends DBBrokerTemplate {
// Datatype and Size
String type = resultSet.getString("Type").toLowerCase();
String datatype;
Integer size = null;
if (type.startsWith("int")) {
datatype = ADQL.INTEGER;
} else if (type.startsWith("smallint")) {
datatype = ADQL.SMALLINT;
} else if (type.startsWith("bigint")) {
datatype = ADQL.BIGINT;
} else if (type.startsWith("float")) {
datatype = ADQL.REAL;
} else if (type.startsWith("char")) {
int beginIndex = type.indexOf('(');
int endIndex = type.indexOf(')');
size = Integer.parseInt(type.substring(beginIndex + 1, endIndex));
datatype = ADQL.CHAR;
} else if (type.startsWith("varchar")) {
int beginIndex = type.indexOf('(');
int endIndex = type.indexOf(')');
size = Integer.parseInt(type.substring(beginIndex + 1, endIndex));
datatype = ADQL.VARCHAR;
} else if (type.contains("timestamp")) {
datatype = ADQL.TIMESTAMP;
} else {
datatype = ADQL.getDataType(type);
}
String datatype = TypesMapping.getADQLTypeFromMySQLType(type);
Integer size = getSize(type);
cm.put(Column.DATATYPE_KEY, datatype);
cm.put(Column.SIZE_KEY, size);
......@@ -160,17 +148,10 @@ public class MySQLDBBroker extends DBBrokerTemplate {
querySb.append(pm.getName());
querySb.append(" ");
Class type = pm.getType();
if (type == String.class) {
querySb.append("VARCHAR(");
querySb.append(pm.getSize());
querySb.append(")");
} else if (type == Integer.class || type == Boolean.class) {
querySb.append("INTEGER");
} else if (type == Long.class) {
querySb.append("BIGINT");
} else {
throw new UnsupportedOperationException("Column type " + type.getCanonicalName() + " not supported yet!");
String mySQLType = TypesMapping.getMySQLTypeFromADQLType(pm.getType());
querySb.append(mySQLType);
if (pm.getType().equals(ADQL.VARCHAR.name()) || pm.getType().equals(ADQL.CHAR.name())) {
appendSize(querySb, pm.getSize());
}
if (pm.isNullable()) {
......
......@@ -29,6 +29,7 @@ import it.inaf.ia2.tsm.datalayer.ADQL;
import it.inaf.ia2.tsm.datalayer.DBBrokerTemplate;
import it.inaf.ia2.tsm.model.PropertyModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.sql.Connection;
import java.sql.ResultSet;
import java.sql.SQLException;
......@@ -77,19 +78,11 @@ public class PostgresDBBroker extends DBBrokerTemplate {
querySb.append(pm.getName());
querySb.append(" ");
Class type = pm.getType();
if (type == String.class) {
querySb.append("character varying(");
querySb.append(pm.getSize());
querySb.append(")");
} else if (type == Integer.class) {
querySb.append("integer");
} else if (type == Boolean.class) {
querySb.append("boolean");
} else if (type == Long.class) {
querySb.append("bigint");
} else {
throw new UnsupportedOperationException("Column type " + type.getCanonicalName() + " not supported yet!");
String pgsqlType = TypesMapping.getPostgresSQLTypeFromADQLType(pm.getType());
querySb.append(pgsqlType);
if (pm.getType().equals(ADQL.VARCHAR.name())
|| pm.getType().equals(ADQL.CHAR.name())) {
appendSize(querySb, pm.getSize());
}
if (pm.isNullable()) {
......@@ -189,25 +182,9 @@ public class PostgresDBBroker extends DBBrokerTemplate {
String datatype;
String type = resultSet.getString("data_type");
if (type.startsWith("int")) {
datatype = ADQL.INTEGER;
} else if (type.startsWith("smallint")) {
datatype = ADQL.SMALLINT;
} else if (type.startsWith("bigint")) {
datatype = ADQL.BIGINT;
} else if (type.startsWith("double") || type.startsWith("real"