Commit 243decdb authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Consistency checking improvements

parent 4b9b9bba
...@@ -44,6 +44,9 @@ public class Column extends ChildEntity<Table> { ...@@ -44,6 +44,9 @@ public class Column extends ChildEntity<Table> {
public final static String UCD_KEY = "ucd"; public final static String UCD_KEY = "ucd";
public final static String UNIT_KEY = "unit"; public final static String UNIT_KEY = "unit";
// Original datatype (computed from information_schema data), used for consistency checking
public final static String ORIGINAL_DATATYPE_KEY = "original_datatype";
private static final long serialVersionUID = 9175956487892235521L; private static final long serialVersionUID = 9175956487892235521L;
private static final Logger LOG = LoggerFactory.getLogger(Column.class); private static final Logger LOG = LoggerFactory.getLogger(Column.class);
...@@ -51,15 +54,17 @@ public class Column extends ChildEntity<Table> { ...@@ -51,15 +54,17 @@ public class Column extends ChildEntity<Table> {
private Key foreignKey; private Key foreignKey;
private Table parentTable; private Table parentTable;
private boolean mandatory;
private Column() { private Column() {
// for serialization // for serialization
super(); super();
} }
protected Column(TapSchema tapSchema, Table table, String columnName) { protected Column(TapSchema tapSchema, Table table, String columnName, boolean mandatory) {
super(tapSchema, tapSchema.getTableModel(TapSchema.COLUMNS_TABLE), table.getColumnMetadata(columnName)); super(tapSchema, tapSchema.getTableModel(TapSchema.COLUMNS_TABLE), table.getColumnMetadata(columnName));
parentTable = table; parentTable = table;
this.mandatory = mandatory;
setStatus(Status.LOADED); setStatus(Status.LOADED);
} }
...@@ -114,6 +119,10 @@ public class Column extends ChildEntity<Table> { ...@@ -114,6 +119,10 @@ public class Column extends ChildEntity<Table> {
return (boolean) getMetadata(PRIMARY_KEY); return (boolean) getMetadata(PRIMARY_KEY);
} }
public boolean isMandatory() {
return mandatory;
}
@Override @Override
public int hashCode() { public int hashCode() {
int hash = 7; int hash = 7;
......
...@@ -24,7 +24,6 @@ package it.inaf.ia2.tsm; ...@@ -24,7 +24,6 @@ package it.inaf.ia2.tsm;
import java.io.Serializable; import java.io.Serializable;
import java.util.Objects; import java.util.Objects;
import java.util.regex.Pattern;
/** /**
* *
...@@ -47,6 +46,12 @@ public class ColumnHolder implements Serializable { ...@@ -47,6 +46,12 @@ public class ColumnHolder implements Serializable {
this.columnName = columnName; this.columnName = columnName;
} }
public ColumnHolder(Column column) {
this.schemaName = column.getParent().getParent().getName();
this.tableName = column.getParent().getName();
this.columnName = column.getName();
}
public String getSchemaName() { public String getSchemaName() {
return schemaName; return schemaName;
} }
......
...@@ -53,9 +53,13 @@ public class ConsistencyChecks implements Serializable { ...@@ -53,9 +53,13 @@ public class ConsistencyChecks implements Serializable {
private final Map<String, Set<String>> tablesToAdd; private final Map<String, Set<String>> tablesToAdd;
private final Map<ColumnHolder, ColumnModel> missingColumns; private final Map<ColumnHolder, ColumnModel> missingColumns;
private final Set<ColumnHolder> columnsToAdd; private final Set<ColumnHolder> columnsToAdd;
private final List<WrongDataType> wrongDataTypes;
private boolean missingObscore; private boolean missingObscore;
private boolean obscoreToAdd; private boolean obscoreToAdd;
// This will not consider an inconsistency: it is only used to display a warning
private final Set<ColumnHolder> unaddedOptionalColumns;
public ConsistencyChecks() { public ConsistencyChecks() {
inconsistencies = new ArrayList<>(); inconsistencies = new ArrayList<>();
unexisingSchemas = new HashSet<>(); unexisingSchemas = new HashSet<>();
...@@ -66,6 +70,8 @@ public class ConsistencyChecks implements Serializable { ...@@ -66,6 +70,8 @@ public class ConsistencyChecks implements Serializable {
tablesToAdd = new HashMap<>(); tablesToAdd = new HashMap<>();
missingColumns = new HashMap<>(); missingColumns = new HashMap<>();
columnsToAdd = new HashSet<>(); columnsToAdd = new HashSet<>();
unaddedOptionalColumns = new HashSet<>();
wrongDataTypes = new ArrayList<>();
} }
public void addInconsistency(InconsistentColumnProperty problemDescription) { public void addInconsistency(InconsistentColumnProperty problemDescription) {
...@@ -182,11 +188,44 @@ public class ConsistencyChecks implements Serializable { ...@@ -182,11 +188,44 @@ public class ConsistencyChecks implements Serializable {
this.obscoreToAdd = obscoreToAdd; this.obscoreToAdd = obscoreToAdd;
} }
public void addUnaddedOptionalColumn(ColumnHolder columnHolder) {
unaddedOptionalColumns.add(columnHolder);
}
public Set<ColumnHolder> getUnaddedOptionalColumns() {
return unaddedOptionalColumns;
}
public void addWrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
// If datatype needs to be changed inconsistency on it doesn't make sense anymore.
Iterator<InconsistentColumnProperty> ite = inconsistencies.iterator();
while (ite.hasNext()) {
InconsistentColumnProperty inconsistency = ite.next();
if (inconsistency.getColumnHolder().equals(columnHolder)
&& (inconsistency.getKey().equals(Column.DATATYPE_KEY)
|| inconsistency.getKey().equals(Column.SIZE_KEY)
|| inconsistency.getKey().equals(Column.ARRAYSIZE_KEY))) {
ite.remove();
}
}
WrongDataType wdt = new WrongDataType(columnHolder, wrongDataType, correctDataType, adqlCorrectDataType, size);
wrongDataTypes.add(wdt);
}
public List<WrongDataType> getWrongDataTypes() {
return wrongDataTypes;
}
public boolean isInconsistent() { public boolean isInconsistent() {
return !inconsistencies.isEmpty() return !inconsistencies.isEmpty() || !wrongDataTypes.isEmpty()
|| !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty() || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty()
|| !missingTables.isEmpty() || !missingColumns.isEmpty() || !missingTables.isEmpty() || !missingColumns.isEmpty()
|| !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty() || !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty()
|| obscoreToAdd || missingObscore; || obscoreToAdd || missingObscore;
} }
public boolean isHasWarnings() {
return !unaddedOptionalColumns.isEmpty();
}
} }
...@@ -32,8 +32,7 @@ public class InconsistentColumnProperty implements Serializable { ...@@ -32,8 +32,7 @@ public class InconsistentColumnProperty implements Serializable {
private static final long serialVersionUID = -5145865322582594970L; private static final long serialVersionUID = -5145865322582594970L;
private String tableCompleteName; private ColumnHolder columnHolder;
private String columnName;
private String key; private String key;
private Object currentValue; private Object currentValue;
private Object correctValue; private Object correctValue;
...@@ -41,22 +40,29 @@ public class InconsistentColumnProperty implements Serializable { ...@@ -41,22 +40,29 @@ public class InconsistentColumnProperty implements Serializable {
private InconsistentColumnProperty() { private InconsistentColumnProperty() {
} }
public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) { public InconsistentColumnProperty(ColumnHolder columnHolder, String key, Object currentValue, Object correctValue) {
this.tableCompleteName = tableCompleteName; this.columnHolder = columnHolder;
this.columnName = columnName;
this.key = key; this.key = key;
this.currentValue = currentValue; this.currentValue = currentValue;
this.correctValue = correctValue; this.correctValue = correctValue;
} }
public ColumnHolder getColumnHolder() {
return columnHolder;
}
public void setColumnHolder(ColumnHolder columnHolder) {
this.columnHolder = columnHolder;
}
public String getTableCompleteName() { public String getTableCompleteName() {
return tableCompleteName; return String.format("%s.%s", columnHolder.getSchemaName(), columnHolder.getTableName());
} }
public String getColumnName() { public String getColumnName() {
return columnName; return columnHolder.getColumnName();
} }
public String getKey() { public String getKey() {
return key; return key;
} }
......
...@@ -52,12 +52,15 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu ...@@ -52,12 +52,15 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
private String simpleName; private String simpleName;
private Schema parentSchema; private Schema parentSchema;
// WARNING: this is different from tableModel inherited field.
private TableModel tableTableModel;
private Table() { private Table() {
// for serialization // for serialization
super(); super();
} }
private TableModel getModel() { private TableModel getTableTableModel() {
if (tapSchema.getName().equals(parentSchema.getName())) { if (tapSchema.getName().equals(parentSchema.getName())) {
return tapSchema.getTapSchemaModel().getTable(simpleName); return tapSchema.getTapSchemaModel().getTable(simpleName);
} }
...@@ -77,7 +80,9 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu ...@@ -77,7 +80,9 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
columns = new TreeMap<>(String.CASE_INSENSITIVE_ORDER); columns = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
DBBroker broker = tapSchema.getDBBroker(schema.getName()); DBBroker broker = tapSchema.getDBBroker(schema.getName());
columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, getModel(), tapSchema.getVersion()); tableTableModel = getTableTableModel();
columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, tableTableModel, tapSchema.getDataTypeMode());
for (Map.Entry<String, Map<String, Object>> entry : columnsMetadata.entrySet()) { for (Map.Entry<String, Map<String, Object>> entry : columnsMetadata.entrySet()) {
// Adding table names to columns metadata // Adding table names to columns metadata
entry.getValue().put(Column.TABLE_NAME_KEY, schema.getName() + "." + tableSimpleName); entry.getValue().put(Column.TABLE_NAME_KEY, schema.getName() + "." + tableSimpleName);
...@@ -103,6 +108,13 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu ...@@ -103,6 +108,13 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
return getValue(TABLE_NAME_KEY, String.class); return getValue(TABLE_NAME_KEY, String.class);
} }
private boolean isMandatory(String columnName) {
if (tableTableModel != null) {
return tableTableModel.get(columnName).isMandatory();
}
return false;
}
/** /**
* {@inheritDoc } * {@inheritDoc }
*/ */
...@@ -117,7 +129,7 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu ...@@ -117,7 +129,7 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
} else { } else {
Column column = columns.get(columnName); Column column = columns.get(columnName);
if (column == null) { if (column == null) {
column = new Column(tapSchema, this, columnName); column = new Column(tapSchema, this, columnName, isMandatory(columnName));
columns.put(columnName, column); columns.put(columnName, column);
column.setStatus(Status.ADDED_NOT_PERSISTED); column.setStatus(Status.ADDED_NOT_PERSISTED);
} else { } else {
......
...@@ -28,10 +28,10 @@ import static it.inaf.ia2.tsm.TapSchema.KEY_COLUMNS_TABLE; ...@@ -28,10 +28,10 @@ import static it.inaf.ia2.tsm.TapSchema.KEY_COLUMNS_TABLE;
import static it.inaf.ia2.tsm.TapSchema.SCHEMAS_TABLE; import static it.inaf.ia2.tsm.TapSchema.SCHEMAS_TABLE;
import static it.inaf.ia2.tsm.TapSchema.TABLES_TABLE; import static it.inaf.ia2.tsm.TapSchema.TABLES_TABLE;
import it.inaf.ia2.tsm.datalayer.DBBroker; import it.inaf.ia2.tsm.datalayer.DBBroker;
import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import it.inaf.ia2.tsm.model.ColumnModel; import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.SchemaModel; import it.inaf.ia2.tsm.model.SchemaModel;
import it.inaf.ia2.tsm.model.TableModel; import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.TypeMapping;
import it.inaf.ia2.tsm.model.TypesMapping; import it.inaf.ia2.tsm.model.TypesMapping;
import java.sql.SQLException; import java.sql.SQLException;
import java.util.ArrayList; import java.util.ArrayList;
...@@ -117,15 +117,27 @@ public class TapSchemaLoader { ...@@ -117,15 +117,27 @@ public class TapSchemaLoader {
Column column = table.getChild(columnModel.getName()); Column column = table.getChild(columnModel.getName());
if (columnModel.isMandatory() && column == null) { ColumnHolder ch = new ColumnHolder(schemaName,
table.getName(), columnModel.getName());
ColumnHolder ch = new ColumnHolder(schemaName, if (column == null) {
table.getName(), columnModel.getName()); if (columnModel.isMandatory()) {
if (table.isAddable(columnModel.getName())) {
if (table.isAddable(columnModel.getName())) { consistencyChecks.addColumnToAdd(ch);
consistencyChecks.addColumnToAdd(ch); } else {
} else { consistencyChecks.addMissingColumn(ch, columnModel);
consistencyChecks.addMissingColumn(ch, columnModel); }
} else if (table.isAddable(columnModel.getName())) {
consistencyChecks.addUnaddedOptionalColumn(ch);
}
} else {
// Data type checking
String originalDataType = (String) column.getMetadata(Column.ORIGINAL_DATATYPE_KEY);
String modelDataType = (String) column.getMetadata(Column.DATATYPE_KEY);
TypeMapping originalType = TypesMapping.getTypeMapping(originalDataType, tapSchema.getDataTypeMode());
TypeMapping modelType = TypesMapping.getTypeMapping(modelDataType, tapSchema.getDataTypeMode());
if (originalType.getJavaType() != modelType.getJavaType()) {
consistencyChecks.addWrongDataType(ch, originalDataType, modelDataType, columnModel.getType(), columnModel.getSize());
} }
} }
} }
...@@ -203,8 +215,7 @@ public class TapSchemaLoader { ...@@ -203,8 +215,7 @@ public class TapSchemaLoader {
Object correctValue = getCorrectValue(column, key); Object correctValue = getCorrectValue(column, key);
if (!Objects.equals(savedValue, correctValue)) { if (!Objects.equals(savedValue, correctValue)) {
InconsistentColumnProperty inconsistentValue = new InconsistentColumnProperty( InconsistentColumnProperty inconsistentValue = new InconsistentColumnProperty(
column.getTableCompleteName(), new ColumnHolder(column),
column.getName(),
key, key,
savedValue, savedValue,
correctValue correctValue
......
...@@ -47,15 +47,7 @@ public class TapSchemaMender { ...@@ -47,15 +47,7 @@ public class TapSchemaMender {
} }
public static void amendTapSchema(TapSchema tapSchema) throws SQLException { public static void amendTapSchema(TapSchema tapSchema) throws SQLException {
int counter = 0; new TapSchemaMender(tapSchema).amendTapSchema();
while (tapSchema.getConsistencyChecks() != null && tapSchema.getConsistencyChecks().isInconsistent()) {
new TapSchemaMender(tapSchema).amendTapSchema();
tapSchema.load();
counter++;
if (counter > 20) {
throw new RuntimeException("Unable to amend TAP_SCHEMA");
}
}
} }
private void amendTapSchema() throws SQLException { private void amendTapSchema() throws SQLException {
...@@ -63,6 +55,7 @@ public class TapSchemaMender { ...@@ -63,6 +55,7 @@ public class TapSchemaMender {
fixObscore(); fixObscore();
createMissingTables(); createMissingTables();
createMissingColumns(); createMissingColumns();
fixDataTypes();
deleteUnexistingEntities(); deleteUnexistingEntities();
addUnaddedTables(); addUnaddedTables();
addUnaddedColumns(); addUnaddedColumns();
...@@ -107,6 +100,16 @@ public class TapSchemaMender { ...@@ -107,6 +100,16 @@ public class TapSchemaMender {
} }
} }
private void fixDataTypes() throws SQLException {
for (WrongDataType wrongDataType : consistencyChecks.getWrongDataTypes()) {
ColumnHolder columnHolder = wrongDataType.getColumnHolder();
DBBroker broker = tapSchema.getDBBroker(columnHolder.getSchemaName());
broker.alterDataType(columnHolder.getSchemaName(),
columnHolder.getTableName(), columnHolder.getColumnName(),
wrongDataType.getAdqlCorrectDataType(), wrongDataType.getSize());
}
}
private void deleteUnexistingEntities() throws SQLException { private void deleteUnexistingEntities() throws SQLException {
String tapSchemaName = tapSchema.getName(); String tapSchemaName = tapSchema.getName();
DBBroker tapSchemaDBBroker = tapSchema.getTapSchemaDBBroker(); DBBroker tapSchemaDBBroker = tapSchema.getTapSchemaDBBroker();
......
/*
* _____________________________________________________________________________
*
* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
* Trieste INAF - IA2 Italian Center for Astronomical Archives
* _____________________________________________________________________________
*
* Copyright (C) 2017 Istituto Nazionale di Astrofisica
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License Version 3 as published by the
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package it.inaf.ia2.tsm;
import java.io.Serializable;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class WrongDataType implements Serializable {
private static final long serialVersionUID = 1541867434766884291L;
private ColumnHolder columnHolder;
private String wrongDataType;
private String correctDataType;
private String adqlCorrectDataType;
private Integer size;
public WrongDataType() {
}
public WrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
this.columnHolder = columnHolder;
this.wrongDataType = wrongDataType;
this.correctDataType = correctDataType;
this.adqlCorrectDataType = adqlCorrectDataType;
this.size = size;
}
public ColumnHolder getColumnHolder() {
return columnHolder;
}
public void setColumnHolder(ColumnHolder columnHolder) {
this.columnHolder = columnHolder;
}
public String getWrongDataType() {
return wrongDataType;
}
public void setWrongDataType(String wrongDataType) {
this.wrongDataType = wrongDataType;
}
public String getCorrectDataType() {
return correctDataType;
}
public void setCorrectDataType(String correctDataType) {
this.correctDataType = correctDataType;
}
public String getAdqlCorrectDataType() {
return adqlCorrectDataType;
}
public void setAdqlCorrectDataType(String adqlCorrectDataType) {
this.adqlCorrectDataType = adqlCorrectDataType;
}
public Integer getSize() {
return size;
}
public void setSize(Integer size) {
this.size = size;
}
}
...@@ -56,7 +56,7 @@ public interface DBBroker { ...@@ -56,7 +56,7 @@ public interface DBBroker {
List<String> getAllColumnsNames(String schemaName, String tableName) throws SQLException; List<String> getAllColumnsNames(String schemaName, String tableName) throws SQLException;
Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, String tapSchemaVersion) throws SQLException; Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException;
List<Key> getKeys(TapSchema tapSchema, String schemaName) throws SQLException; List<Key> getKeys(TapSchema tapSchema, String schemaName) throws SQLException;
...@@ -89,4 +89,6 @@ public interface DBBroker { ...@@ -89,4 +89,6 @@ public interface DBBroker {
Set<String> getKeysToRemoveFromUnexistingColumn(String tapSchemaName, ColumnHolder unexistingColumn) throws SQLException; Set<String> getKeysToRemoveFromUnexistingColumn(String tapSchemaName, ColumnHolder unexistingColumn) throws SQLException;
void updateTapSchemaColumnValue(String tapSchemaName, String completeTableName, String columnName, String key, Object value) throws SQLException; void updateTapSchemaColumnValue(String tapSchemaName, String completeTableName, String columnName, String key, Object value) throws SQLException;
void alterDataType(String schemaName, String tableName, String columnName, String adqlDataType, Integer size) throws SQLException;
} }
...@@ -781,17 +781,25 @@ public abstract class DBBrokerTemplate implements DBBroker { ...@@ -781,17 +781,25 @@ public abstract class DBBrokerTemplate implements DBBroker {
protected abstract Map<String, Map<String, Object>> getAllColumnsOriginalMetadata(String schemaName, String tableName) throws SQLException; protected abstract Map<String, Map<String, Object>> getAllColumnsOriginalMetadata(String schemaName, String tableName) throws SQLException;
@Override @Override
public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, String tapSchemaVersion) throws SQLException { public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException {
Map<String, Map<String, Object>> metadata = getAllColumnsOriginalMetadata(schemaName, tableName