Commit 243decdb authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Consistency checking improvements

parent 4b9b9bba
......@@ -44,6 +44,9 @@ public class Column extends ChildEntity<Table> {
public final static String UCD_KEY = "ucd";
public final static String UNIT_KEY = "unit";
// Original datatype (computed from information_schema data), used for consistency checking
public final static String ORIGINAL_DATATYPE_KEY = "original_datatype";
private static final long serialVersionUID = 9175956487892235521L;
private static final Logger LOG = LoggerFactory.getLogger(Column.class);
......@@ -51,15 +54,17 @@ public class Column extends ChildEntity<Table> {
private Key foreignKey;
private Table parentTable;
private boolean mandatory;
private Column() {
// for serialization
super();
}
protected Column(TapSchema tapSchema, Table table, String columnName) {
protected Column(TapSchema tapSchema, Table table, String columnName, boolean mandatory) {
super(tapSchema, tapSchema.getTableModel(TapSchema.COLUMNS_TABLE), table.getColumnMetadata(columnName));
parentTable = table;
this.mandatory = mandatory;
setStatus(Status.LOADED);
}
......@@ -114,6 +119,10 @@ public class Column extends ChildEntity<Table> {
return (boolean) getMetadata(PRIMARY_KEY);
}
public boolean isMandatory() {
return mandatory;
}
@Override
public int hashCode() {
int hash = 7;
......
......@@ -24,7 +24,6 @@ package it.inaf.ia2.tsm;
import java.io.Serializable;
import java.util.Objects;
import java.util.regex.Pattern;
/**
*
......@@ -47,6 +46,12 @@ public class ColumnHolder implements Serializable {
this.columnName = columnName;
}
public ColumnHolder(Column column) {
this.schemaName = column.getParent().getParent().getName();
this.tableName = column.getParent().getName();
this.columnName = column.getName();
}
public String getSchemaName() {
return schemaName;
}
......
......@@ -53,9 +53,13 @@ public class ConsistencyChecks implements Serializable {
private final Map<String, Set<String>> tablesToAdd;
private final Map<ColumnHolder, ColumnModel> missingColumns;
private final Set<ColumnHolder> columnsToAdd;
private final List<WrongDataType> wrongDataTypes;
private boolean missingObscore;
private boolean obscoreToAdd;
// This will not consider an inconsistency: it is only used to display a warning
private final Set<ColumnHolder> unaddedOptionalColumns;
public ConsistencyChecks() {
inconsistencies = new ArrayList<>();
unexisingSchemas = new HashSet<>();
......@@ -66,6 +70,8 @@ public class ConsistencyChecks implements Serializable {
tablesToAdd = new HashMap<>();
missingColumns = new HashMap<>();
columnsToAdd = new HashSet<>();
unaddedOptionalColumns = new HashSet<>();
wrongDataTypes = new ArrayList<>();
}
public void addInconsistency(InconsistentColumnProperty problemDescription) {
......@@ -182,11 +188,44 @@ public class ConsistencyChecks implements Serializable {
this.obscoreToAdd = obscoreToAdd;
}
public void addUnaddedOptionalColumn(ColumnHolder columnHolder) {
unaddedOptionalColumns.add(columnHolder);
}
public Set<ColumnHolder> getUnaddedOptionalColumns() {
return unaddedOptionalColumns;
}
public void addWrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
// If datatype needs to be changed inconsistency on it doesn't make sense anymore.
Iterator<InconsistentColumnProperty> ite = inconsistencies.iterator();
while (ite.hasNext()) {
InconsistentColumnProperty inconsistency = ite.next();
if (inconsistency.getColumnHolder().equals(columnHolder)
&& (inconsistency.getKey().equals(Column.DATATYPE_KEY)
|| inconsistency.getKey().equals(Column.SIZE_KEY)
|| inconsistency.getKey().equals(Column.ARRAYSIZE_KEY))) {
ite.remove();
}
}
WrongDataType wdt = new WrongDataType(columnHolder, wrongDataType, correctDataType, adqlCorrectDataType, size);
wrongDataTypes.add(wdt);
}
public List<WrongDataType> getWrongDataTypes() {
return wrongDataTypes;
}
public boolean isInconsistent() {
return !inconsistencies.isEmpty()
return !inconsistencies.isEmpty() || !wrongDataTypes.isEmpty()
|| !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty()
|| !missingTables.isEmpty() || !missingColumns.isEmpty()
|| !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty()
|| obscoreToAdd || missingObscore;
}
public boolean isHasWarnings() {
return !unaddedOptionalColumns.isEmpty();
}
}
......@@ -32,8 +32,7 @@ public class InconsistentColumnProperty implements Serializable {
private static final long serialVersionUID = -5145865322582594970L;
private String tableCompleteName;
private String columnName;
private ColumnHolder columnHolder;
private String key;
private Object currentValue;
private Object correctValue;
......@@ -41,22 +40,29 @@ public class InconsistentColumnProperty implements Serializable {
private InconsistentColumnProperty() {
}
public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) {
this.tableCompleteName = tableCompleteName;
this.columnName = columnName;
public InconsistentColumnProperty(ColumnHolder columnHolder, String key, Object currentValue, Object correctValue) {
this.columnHolder = columnHolder;
this.key = key;
this.currentValue = currentValue;
this.correctValue = correctValue;
}
public ColumnHolder getColumnHolder() {
return columnHolder;
}
public void setColumnHolder(ColumnHolder columnHolder) {
this.columnHolder = columnHolder;
}
public String getTableCompleteName() {
return tableCompleteName;
return String.format("%s.%s", columnHolder.getSchemaName(), columnHolder.getTableName());
}
public String getColumnName() {
return columnName;
return columnHolder.getColumnName();
}
public String getKey() {
return key;
}
......
......@@ -52,12 +52,15 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
private String simpleName;
private Schema parentSchema;
// WARNING: this is different from tableModel inherited field.
private TableModel tableTableModel;
private Table() {
// for serialization
super();
}
private TableModel getModel() {
private TableModel getTableTableModel() {
if (tapSchema.getName().equals(parentSchema.getName())) {
return tapSchema.getTapSchemaModel().getTable(simpleName);
}
......@@ -77,7 +80,9 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
columns = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
DBBroker broker = tapSchema.getDBBroker(schema.getName());
columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, getModel(), tapSchema.getVersion());
tableTableModel = getTableTableModel();
columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, tableTableModel, tapSchema.getDataTypeMode());
for (Map.Entry<String, Map<String, Object>> entry : columnsMetadata.entrySet()) {
// Adding table names to columns metadata
entry.getValue().put(Column.TABLE_NAME_KEY, schema.getName() + "." + tableSimpleName);
......@@ -103,6 +108,13 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
return getValue(TABLE_NAME_KEY, String.class);
}
private boolean isMandatory(String columnName) {
if (tableTableModel != null) {
return tableTableModel.get(columnName).isMandatory();
}
return false;
}
/**
* {@inheritDoc }
*/
......@@ -117,7 +129,7 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
} else {
Column column = columns.get(columnName);
if (column == null) {
column = new Column(tapSchema, this, columnName);
column = new Column(tapSchema, this, columnName, isMandatory(columnName));
columns.put(columnName, column);
column.setStatus(Status.ADDED_NOT_PERSISTED);
} else {
......
......@@ -28,10 +28,10 @@ import static it.inaf.ia2.tsm.TapSchema.KEY_COLUMNS_TABLE;
import static it.inaf.ia2.tsm.TapSchema.SCHEMAS_TABLE;
import static it.inaf.ia2.tsm.TapSchema.TABLES_TABLE;
import it.inaf.ia2.tsm.datalayer.DBBroker;
import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.SchemaModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypeMapping;
import it.inaf.ia2.tsm.model.TypesMapping;
import java.sql.SQLException;
import java.util.ArrayList;
......@@ -117,15 +117,27 @@ public class TapSchemaLoader {
Column column = table.getChild(columnModel.getName());
if (columnModel.isMandatory() && column == null) {
ColumnHolder ch = new ColumnHolder(schemaName,
table.getName(), columnModel.getName());
ColumnHolder ch = new ColumnHolder(schemaName,
table.getName(), columnModel.getName());
if (table.isAddable(columnModel.getName())) {
consistencyChecks.addColumnToAdd(ch);
} else {
consistencyChecks.addMissingColumn(ch, columnModel);
if (column == null) {
if (columnModel.isMandatory()) {
if (table.isAddable(columnModel.getName())) {
consistencyChecks.addColumnToAdd(ch);
} else {
consistencyChecks.addMissingColumn(ch, columnModel);
}
} else if (table.isAddable(columnModel.getName())) {
consistencyChecks.addUnaddedOptionalColumn(ch);
}
} else {
// Data type checking
String originalDataType = (String) column.getMetadata(Column.ORIGINAL_DATATYPE_KEY);
String modelDataType = (String) column.getMetadata(Column.DATATYPE_KEY);
TypeMapping originalType = TypesMapping.getTypeMapping(originalDataType, tapSchema.getDataTypeMode());
TypeMapping modelType = TypesMapping.getTypeMapping(modelDataType, tapSchema.getDataTypeMode());
if (originalType.getJavaType() != modelType.getJavaType()) {
consistencyChecks.addWrongDataType(ch, originalDataType, modelDataType, columnModel.getType(), columnModel.getSize());
}
}
}
......@@ -203,8 +215,7 @@ public class TapSchemaLoader {
Object correctValue = getCorrectValue(column, key);
if (!Objects.equals(savedValue, correctValue)) {
InconsistentColumnProperty inconsistentValue = new InconsistentColumnProperty(
column.getTableCompleteName(),
column.getName(),
new ColumnHolder(column),
key,
savedValue,
correctValue
......
......@@ -47,15 +47,7 @@ public class TapSchemaMender {
}
public static void amendTapSchema(TapSchema tapSchema) throws SQLException {
int counter = 0;
while (tapSchema.getConsistencyChecks() != null && tapSchema.getConsistencyChecks().isInconsistent()) {
new TapSchemaMender(tapSchema).amendTapSchema();
tapSchema.load();
counter++;
if (counter > 20) {
throw new RuntimeException("Unable to amend TAP_SCHEMA");
}
}
new TapSchemaMender(tapSchema).amendTapSchema();
}
private void amendTapSchema() throws SQLException {
......@@ -63,6 +55,7 @@ public class TapSchemaMender {
fixObscore();
createMissingTables();
createMissingColumns();
fixDataTypes();
deleteUnexistingEntities();
addUnaddedTables();
addUnaddedColumns();
......@@ -107,6 +100,16 @@ public class TapSchemaMender {
}
}
private void fixDataTypes() throws SQLException {
for (WrongDataType wrongDataType : consistencyChecks.getWrongDataTypes()) {
ColumnHolder columnHolder = wrongDataType.getColumnHolder();
DBBroker broker = tapSchema.getDBBroker(columnHolder.getSchemaName());
broker.alterDataType(columnHolder.getSchemaName(),
columnHolder.getTableName(), columnHolder.getColumnName(),
wrongDataType.getAdqlCorrectDataType(), wrongDataType.getSize());
}
}
private void deleteUnexistingEntities() throws SQLException {
String tapSchemaName = tapSchema.getName();
DBBroker tapSchemaDBBroker = tapSchema.getTapSchemaDBBroker();
......
/*
* _____________________________________________________________________________
*
* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
* Trieste INAF - IA2 Italian Center for Astronomical Archives
* _____________________________________________________________________________
*
* Copyright (C) 2017 Istituto Nazionale di Astrofisica
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License Version 3 as published by the
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package it.inaf.ia2.tsm;
import java.io.Serializable;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class WrongDataType implements Serializable {
private static final long serialVersionUID = 1541867434766884291L;
private ColumnHolder columnHolder;
private String wrongDataType;
private String correctDataType;
private String adqlCorrectDataType;
private Integer size;
public WrongDataType() {
}
public WrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
this.columnHolder = columnHolder;
this.wrongDataType = wrongDataType;
this.correctDataType = correctDataType;
this.adqlCorrectDataType = adqlCorrectDataType;
this.size = size;
}
public ColumnHolder getColumnHolder() {
return columnHolder;
}
public void setColumnHolder(ColumnHolder columnHolder) {
this.columnHolder = columnHolder;
}
public String getWrongDataType() {
return wrongDataType;
}
public void setWrongDataType(String wrongDataType) {
this.wrongDataType = wrongDataType;
}
public String getCorrectDataType() {
return correctDataType;
}
public void setCorrectDataType(String correctDataType) {
this.correctDataType = correctDataType;
}
public String getAdqlCorrectDataType() {
return adqlCorrectDataType;
}
public void setAdqlCorrectDataType(String adqlCorrectDataType) {
this.adqlCorrectDataType = adqlCorrectDataType;
}
public Integer getSize() {
return size;
}
public void setSize(Integer size) {
this.size = size;
}
}
......@@ -56,7 +56,7 @@ public interface DBBroker {
List<String> getAllColumnsNames(String schemaName, String tableName) throws SQLException;
Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, String tapSchemaVersion) throws SQLException;
Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException;
List<Key> getKeys(TapSchema tapSchema, String schemaName) throws SQLException;
......@@ -89,4 +89,6 @@ public interface DBBroker {
Set<String> getKeysToRemoveFromUnexistingColumn(String tapSchemaName, ColumnHolder unexistingColumn) throws SQLException;
void updateTapSchemaColumnValue(String tapSchemaName, String completeTableName, String columnName, String key, Object value) throws SQLException;
void alterDataType(String schemaName, String tableName, String columnName, String adqlDataType, Integer size) throws SQLException;
}
......@@ -781,17 +781,25 @@ public abstract class DBBrokerTemplate implements DBBroker {
protected abstract Map<String, Map<String, Object>> getAllColumnsOriginalMetadata(String schemaName, String tableName) throws SQLException;
@Override
public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, String tapSchemaVersion) throws SQLException {
public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException {
Map<String, Map<String, Object>> metadata = getAllColumnsOriginalMetadata(schemaName, tableName);
// Overriding data type from models
// Special behavior for data type
if (tableModel != null) {
for (Map.Entry<String, Map<String, Object>> entry : metadata.entrySet()) {
String columnName = entry.getKey();
// Saving original data type (used for consistency checking)
Map<String, Object> columnMetadata = entry.getValue();
String originalDataType = (String) columnMetadata.get(Column.DATATYPE_KEY);
columnMetadata.put(Column.ORIGINAL_DATATYPE_KEY, originalDataType);
// Override data type using model definition
String adqlType = tableModel.get(columnName).getType();
String tapDataType = TypesMapping.getDataType(adqlType, dataTypeMode);
entry.getValue().put(Column.DATATYPE_KEY, tapDataType);
String definedDataType = TypesMapping.getDataType(adqlType, dataTypeMode);
columnMetadata.put(Column.DATATYPE_KEY, definedDataType);
}
}
......@@ -1005,4 +1013,41 @@ public abstract class DBBrokerTemplate implements DBBroker {
ps.executeUpdate();
}
}
protected String getEnumDefinition(List<String> enumValues) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (String enumValue : enumValues) {
if (!first) {
sb.append(',');
}
sb.append(String.format("'%s'", enumValue));
first = false;
}
return sb.toString();
}
private String getCheckConstraint(ColumnModel columnModel) {
if (columnModel.getMinValue() != null && columnModel.getMaxValue() != null) {
return String.format("CHECK (%s >= %s AND %s <= %s)", columnModel.getName(),
columnModel.getMinValue(), columnModel.getName(), columnModel.getMaxValue());
} else if (columnModel.getMinValue() != null && columnModel.getMaxValue() == null) {
return String.format("CHECK (%s >= %s)", columnModel.getName(), columnModel.getMinValue());
} else if (columnModel.getMinValue() == null && columnModel.getMaxValue() != null) {
return String.format("CHECK (%s <= %s)", columnModel.getName(), columnModel.getMaxValue());
}
return null;
}
protected void appendCheckConstraints(StringBuilder querySb, TableModel tableModel) {
for (ColumnModel cm : tableModel.getColumns()) {
String checkConstraint = getCheckConstraint(cm);
if (checkConstraint != null) {
querySb.append(",\n");
querySb.append(checkConstraint);
}
}
}
}
......@@ -50,6 +50,7 @@ import org.slf4j.LoggerFactory;
public class MySQLDBBroker extends DBBrokerTemplate {
private static final Logger LOG = LoggerFactory.getLogger(MySQLDBBroker.class);
private static final int DEFAULT_VARCHAR_SIZE = 255;
public MySQLDBBroker(DataSource dataSource, DataTypeMode mode) {
super(dataSource, '`', mode);
......@@ -170,10 +171,21 @@ public class MySQLDBBroker extends DBBrokerTemplate {
querySb.append(cm.getName());
querySb.append(" ");
String mySQLType = TypesMapping.getMySQLTypeFromADQLType(cm.getType()).toUpperCase();
querySb.append(mySQLType);
if (mySQLType.equals("VARCHAR") || mySQLType.equals("CHAR")) {
appendSize(querySb, cm.getSize());
String mySQLType;
if (cm.getEnumValues() == null) {
mySQLType = TypesMapping.getMySQLTypeFromADQLType(cm.getType()).toUpperCase();
querySb.append(mySQLType);
Integer size = cm.getSize();
if (mySQLType.equals("VARCHAR") && size == null) {
size = DEFAULT_VARCHAR_SIZE;
}
if (size != null) {
appendSize(querySb, size);
}
} else {
querySb.append("ENUM (");
querySb.append(getEnumDefinition(cm.getEnumValues()));
querySb.append(")");
}
if (cm.isNullable()) {
......@@ -183,6 +195,7 @@ public class MySQLDBBroker extends DBBrokerTemplate {
}
}
}
appendCheckConstraints(querySb, tableModel);
querySb.append(")");
......@@ -284,4 +297,21 @@ public class MySQLDBBroker extends DBBrokerTemplate {
protected String getTableTypesQuery(String schemaName) {
return "SELECT table_name, table_type FROM information_schema.tables WHERE table_schema = '" + schemaName + "'";
}
@Override
public void alterDataType(String schemaName, String tableName, String columnName, String adqlDataType, Integer size) throws SQLException {
String mySQLDataType = TypesMapping.getMySQLTypeFromADQLType(adqlDataType);
if (size != null) {
mySQLDataType = String.format("%s(%s)", mySQLDataType, size);
}
String query = String.format("ALTER TABLE %s.%s MODIFY %s %s", escape(schemaName), escape(tableName), escape(columnName), mySQLDataType);
try (Connection connection = dataSource.getConnection();
Statement statement = connection.createStatement()) {
LOG.debug("Executing query: {}", query);
statement.executeUpdate(query);
}
}
}
......@@ -32,6 +32,7 @@ import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.TableModel;