Consistency checking improvements

TASMAN-core/src/main/java/it/inaf/ia2/tsm/Column.java

@@ -44,6 +44,9 @@ public class Column extends ChildEntity<Table> {
     public final static String UCD_KEY = "ucd";
     public final static String UNIT_KEY = "unit";
 
+    // Original datatype (computed from information_schema data), used for consistency checking
+    public final static String ORIGINAL_DATATYPE_KEY = "original_datatype";
+
     private static final long serialVersionUID = 9175956487892235521L;
     private static final Logger LOG = LoggerFactory.getLogger(Column.class);
 
@@ -51,15 +54,17 @@ public class Column extends ChildEntity<Table> {
     private Key foreignKey;
 
     private Table parentTable;
+    private boolean mandatory;
 
     private Column() {
         // for serialization
         super();
     }
 
-    protected Column(TapSchema tapSchema, Table table, String columnName) {
+    protected Column(TapSchema tapSchema, Table table, String columnName, boolean mandatory) {
         super(tapSchema, tapSchema.getTableModel(TapSchema.COLUMNS_TABLE), table.getColumnMetadata(columnName));
         parentTable = table;
+        this.mandatory = mandatory;
         setStatus(Status.LOADED);
     }
 
@@ -114,6 +119,10 @@ public class Column extends ChildEntity<Table> {
         return (boolean) getMetadata(PRIMARY_KEY);
     }
 
+    public boolean isMandatory() {
+        return mandatory;
+    }
+
     @Override
     public int hashCode() {
         int hash = 7;

TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java

@@ -24,7 +24,6 @@ package it.inaf.ia2.tsm;
 
 import java.io.Serializable;
 import java.util.Objects;
-import java.util.regex.Pattern;
 
 /**
  *
@@ -47,6 +46,12 @@ public class ColumnHolder implements Serializable {
         this.columnName = columnName;
     }
 
+    public ColumnHolder(Column column) {
+        this.schemaName = column.getParent().getParent().getName();
+        this.tableName = column.getParent().getName();
+        this.columnName = column.getName();
+    }
+
     public String getSchemaName() {
         return schemaName;
     }

TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java

@@ -53,9 +53,13 @@ public class ConsistencyChecks implements Serializable {
     private final Map<String, Set<String>> tablesToAdd;
     private final Map<ColumnHolder, ColumnModel> missingColumns;
     private final Set<ColumnHolder> columnsToAdd;
+    private final List<WrongDataType> wrongDataTypes;
     private boolean missingObscore;
     private boolean obscoreToAdd;
 
+    // This will not consider an inconsistency: it is only used to display a warning
+    private final Set<ColumnHolder> unaddedOptionalColumns;
+
     public ConsistencyChecks() {
         inconsistencies = new ArrayList<>();
         unexisingSchemas = new HashSet<>();
@@ -66,6 +70,8 @@ public class ConsistencyChecks implements Serializable {
         tablesToAdd = new HashMap<>();
         missingColumns = new HashMap<>();
         columnsToAdd = new HashSet<>();
+        unaddedOptionalColumns = new HashSet<>();
+        wrongDataTypes = new ArrayList<>();
     }
 
     public void addInconsistency(InconsistentColumnProperty problemDescription) {
@@ -182,11 +188,44 @@ public class ConsistencyChecks implements Serializable {
         this.obscoreToAdd = obscoreToAdd;
     }
 
+    public void addUnaddedOptionalColumn(ColumnHolder columnHolder) {
+        unaddedOptionalColumns.add(columnHolder);
+    }
+
+    public Set<ColumnHolder> getUnaddedOptionalColumns() {
+        return unaddedOptionalColumns;
+    }
+
+    public void addWrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
+        // If datatype needs to be changed inconsistency on it doesn't make sense anymore.
+        Iterator<InconsistentColumnProperty> ite = inconsistencies.iterator();
+        while (ite.hasNext()) {
+            InconsistentColumnProperty inconsistency = ite.next();
+            if (inconsistency.getColumnHolder().equals(columnHolder)
+                    && (inconsistency.getKey().equals(Column.DATATYPE_KEY)
+                    || inconsistency.getKey().equals(Column.SIZE_KEY)
+                    || inconsistency.getKey().equals(Column.ARRAYSIZE_KEY))) {
+                ite.remove();
+            }
+        }
+
+        WrongDataType wdt = new WrongDataType(columnHolder, wrongDataType, correctDataType, adqlCorrectDataType, size);
+        wrongDataTypes.add(wdt);
+    }
+
+    public List<WrongDataType> getWrongDataTypes() {
+        return wrongDataTypes;
+    }
+
     public boolean isInconsistent() {
-        return !inconsistencies.isEmpty()
+        return !inconsistencies.isEmpty() || !wrongDataTypes.isEmpty()
                 || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty()
                 || !missingTables.isEmpty() || !missingColumns.isEmpty()
                 || !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty()
                 || obscoreToAdd || missingObscore;
     }
+
+    public boolean isHasWarnings() {
+        return !unaddedOptionalColumns.isEmpty();
+    }
 }

TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java

@@ -32,8 +32,7 @@ public class InconsistentColumnProperty implements Serializable {
 
     private static final long serialVersionUID = -5145865322582594970L;
 
-    private String tableCompleteName;
-    private String columnName;
+    private ColumnHolder columnHolder;
     private String key;
     private Object currentValue;
     private Object correctValue;
@@ -41,22 +40,29 @@ public class InconsistentColumnProperty implements Serializable {
     private InconsistentColumnProperty() {
     }
 
-    public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) {
-        this.tableCompleteName = tableCompleteName;
-        this.columnName = columnName;
+    public InconsistentColumnProperty(ColumnHolder columnHolder, String key, Object currentValue, Object correctValue) {
+        this.columnHolder = columnHolder;
         this.key = key;
         this.currentValue = currentValue;
         this.correctValue = correctValue;
     }
 
+    public ColumnHolder getColumnHolder() {
+        return columnHolder;
+    }
+
+    public void setColumnHolder(ColumnHolder columnHolder) {
+        this.columnHolder = columnHolder;
+    }
+
     public String getTableCompleteName() {
-        return tableCompleteName;
+        return String.format("%s.%s", columnHolder.getSchemaName(), columnHolder.getTableName());
     }
 
     public String getColumnName() {
-        return columnName;
+        return columnHolder.getColumnName();
     }
-
+    
     public String getKey() {
         return key;
     }

TASMAN-core/src/main/java/it/inaf/ia2/tsm/Table.java

@@ -52,12 +52,15 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
     private String simpleName;
     private Schema parentSchema;
 
+    // WARNING: this is different from tableModel inherited field.
+    private TableModel tableTableModel;
+
     private Table() {
         // for serialization
         super();
     }
 
-    private TableModel getModel() {
+    private TableModel getTableTableModel() {
         if (tapSchema.getName().equals(parentSchema.getName())) {
             return tapSchema.getTapSchemaModel().getTable(simpleName);
         }
@@ -77,7 +80,9 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
         columns = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
 
         DBBroker broker = tapSchema.getDBBroker(schema.getName());
-        columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, getModel(), tapSchema.getVersion());
+        tableTableModel = getTableTableModel();
+        columnsMetadata = broker.getAllColumnsMetadata(schema.getName(), tableSimpleName, tableTableModel, tapSchema.getDataTypeMode());
+
         for (Map.Entry<String, Map<String, Object>> entry : columnsMetadata.entrySet()) {
             // Adding table names to columns metadata
             entry.getValue().put(Column.TABLE_NAME_KEY, schema.getName() + "." + tableSimpleName);
@@ -103,6 +108,13 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
         return getValue(TABLE_NAME_KEY, String.class);
     }
 
+    private boolean isMandatory(String columnName) {
+        if (tableTableModel != null) {
+            return tableTableModel.get(columnName).isMandatory();
+        }
+        return false;
+    }
+
     /**
      * {@inheritDoc }
      */
@@ -117,7 +129,7 @@ public class Table extends ChildEntity<Schema> implements EntitiesContainer<Colu
         } else {
             Column column = columns.get(columnName);
             if (column == null) {
-                column = new Column(tapSchema, this, columnName);
+                column = new Column(tapSchema, this, columnName, isMandatory(columnName));
                 columns.put(columnName, column);
                 column.setStatus(Status.ADDED_NOT_PERSISTED);
             } else {

TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaLoader.java

@@ -28,10 +28,10 @@ import static it.inaf.ia2.tsm.TapSchema.KEY_COLUMNS_TABLE;
 import static it.inaf.ia2.tsm.TapSchema.SCHEMAS_TABLE;
 import static it.inaf.ia2.tsm.TapSchema.TABLES_TABLE;
 import it.inaf.ia2.tsm.datalayer.DBBroker;
-import it.inaf.ia2.tsm.datalayer.DataTypeMode;
 import it.inaf.ia2.tsm.model.ColumnModel;
 import it.inaf.ia2.tsm.model.SchemaModel;
 import it.inaf.ia2.tsm.model.TableModel;
+import it.inaf.ia2.tsm.model.TypeMapping;
 import it.inaf.ia2.tsm.model.TypesMapping;
 import java.sql.SQLException;
 import java.util.ArrayList;
@@ -117,15 +117,27 @@ public class TapSchemaLoader {
 
                     Column column = table.getChild(columnModel.getName());
 
-                    if (columnModel.isMandatory() && column == null) {
+                    ColumnHolder ch = new ColumnHolder(schemaName,
+                            table.getName(), columnModel.getName());
 
-                        ColumnHolder ch = new ColumnHolder(schemaName,
-                                table.getName(), columnModel.getName());
-
-                        if (table.isAddable(columnModel.getName())) {
-                            consistencyChecks.addColumnToAdd(ch);
-                        } else {
-                            consistencyChecks.addMissingColumn(ch, columnModel);
+                    if (column == null) {
+                        if (columnModel.isMandatory()) {
+                            if (table.isAddable(columnModel.getName())) {
+                                consistencyChecks.addColumnToAdd(ch);
+                            } else {
+                                consistencyChecks.addMissingColumn(ch, columnModel);
+                            }
+                        } else if (table.isAddable(columnModel.getName())) {
+                            consistencyChecks.addUnaddedOptionalColumn(ch);
+                        }
+                    } else {
+                        // Data type checking
+                        String originalDataType = (String) column.getMetadata(Column.ORIGINAL_DATATYPE_KEY);
+                        String modelDataType = (String) column.getMetadata(Column.DATATYPE_KEY);
+                        TypeMapping originalType = TypesMapping.getTypeMapping(originalDataType, tapSchema.getDataTypeMode());
+                        TypeMapping modelType = TypesMapping.getTypeMapping(modelDataType, tapSchema.getDataTypeMode());
+                        if (originalType.getJavaType() != modelType.getJavaType()) {
+                            consistencyChecks.addWrongDataType(ch, originalDataType, modelDataType, columnModel.getType(), columnModel.getSize());
                         }
                     }
                 }
@@ -203,8 +215,7 @@ public class TapSchemaLoader {
                     Object correctValue = getCorrectValue(column, key);
                     if (!Objects.equals(savedValue, correctValue)) {
                         InconsistentColumnProperty inconsistentValue = new InconsistentColumnProperty(
-                                column.getTableCompleteName(),
-                                column.getName(),
+                                new ColumnHolder(column),
                                 key,
                                 savedValue,
                                 correctValue

TASMAN-core/src/main/java/it/inaf/ia2/tsm/TapSchemaMender.java

@@ -47,15 +47,7 @@ public class TapSchemaMender {
     }
 
     public static void amendTapSchema(TapSchema tapSchema) throws SQLException {
-        int counter = 0;
-        while (tapSchema.getConsistencyChecks() != null && tapSchema.getConsistencyChecks().isInconsistent()) {
-            new TapSchemaMender(tapSchema).amendTapSchema();
-            tapSchema.load();
-            counter++;
-            if (counter > 20) {
-                throw new RuntimeException("Unable to amend TAP_SCHEMA");
-            }
-        }
+        new TapSchemaMender(tapSchema).amendTapSchema();
     }
 
     private void amendTapSchema() throws SQLException {
@@ -63,6 +55,7 @@ public class TapSchemaMender {
         fixObscore();
         createMissingTables();
         createMissingColumns();
+        fixDataTypes();
         deleteUnexistingEntities();
         addUnaddedTables();
         addUnaddedColumns();
@@ -107,6 +100,16 @@ public class TapSchemaMender {
         }
     }
 
+    private void fixDataTypes() throws SQLException {
+        for (WrongDataType wrongDataType : consistencyChecks.getWrongDataTypes()) {
+            ColumnHolder columnHolder = wrongDataType.getColumnHolder();
+            DBBroker broker = tapSchema.getDBBroker(columnHolder.getSchemaName());
+            broker.alterDataType(columnHolder.getSchemaName(),
+                    columnHolder.getTableName(), columnHolder.getColumnName(),
+                    wrongDataType.getAdqlCorrectDataType(), wrongDataType.getSize());
+        }
+    }
+
     private void deleteUnexistingEntities() throws SQLException {
         String tapSchemaName = tapSchema.getName();
         DBBroker tapSchemaDBBroker = tapSchema.getTapSchemaDBBroker();

TASMAN-core/src/main/java/it/inaf/ia2/tsm/WrongDataType.java

+/*
+ * _____________________________________________________________________________
+ * 
+ * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
+ * Trieste INAF - IA2 Italian Center for Astronomical Archives
+ * _____________________________________________________________________________
+ * 
+ * Copyright (C) 2017 Istituto Nazionale di Astrofisica
+ * 
+ * This program is free software; you can redistribute it and/or modify it under
+ * the terms of the GNU General Public License Version 3 as published by the
+ * Free Software Foundation.
+ * 
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
+ * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
+ * details.
+ * 
+ * You should have received a copy of the GNU General Public License along with
+ * this program; if not, write to the Free Software Foundation, Inc., 51
+ * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
+ */
+package it.inaf.ia2.tsm;
+
+import java.io.Serializable;
+
+/**
+ *
+ * @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
+ */
+public class WrongDataType implements Serializable {
+
+    private static final long serialVersionUID = 1541867434766884291L;
+
+    private ColumnHolder columnHolder;
+    private String wrongDataType;
+    private String correctDataType;
+    private String adqlCorrectDataType;
+    private Integer size;
+
+    public WrongDataType() {
+    }
+
+    public WrongDataType(ColumnHolder columnHolder, String wrongDataType, String correctDataType, String adqlCorrectDataType, Integer size) {
+        this.columnHolder = columnHolder;
+        this.wrongDataType = wrongDataType;
+        this.correctDataType = correctDataType;
+        this.adqlCorrectDataType = adqlCorrectDataType;
+        this.size = size;
+    }
+
+    public ColumnHolder getColumnHolder() {
+        return columnHolder;
+    }
+
+    public void setColumnHolder(ColumnHolder columnHolder) {
+        this.columnHolder = columnHolder;
+    }
+
+    public String getWrongDataType() {
+        return wrongDataType;
+    }
+
+    public void setWrongDataType(String wrongDataType) {
+        this.wrongDataType = wrongDataType;
+    }
+
+    public String getCorrectDataType() {
+        return correctDataType;
+    }
+
+    public void setCorrectDataType(String correctDataType) {
+        this.correctDataType = correctDataType;
+    }
+
+    public String getAdqlCorrectDataType() {
+        return adqlCorrectDataType;
+    }
+
+    public void setAdqlCorrectDataType(String adqlCorrectDataType) {
+        this.adqlCorrectDataType = adqlCorrectDataType;
+    }
+
+    public Integer getSize() {
+        return size;
+    }
+
+    public void setSize(Integer size) {
+        this.size = size;
+    }
+}

TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBroker.java

@@ -56,7 +56,7 @@ public interface DBBroker {
 
     List<String> getAllColumnsNames(String schemaName, String tableName) throws SQLException;
 
-    Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, String tapSchemaVersion) throws SQLException;
+    Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableSimpleName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException;
 
     List<Key> getKeys(TapSchema tapSchema, String schemaName) throws SQLException;
 
@@ -89,4 +89,6 @@ public interface DBBroker {
     Set<String> getKeysToRemoveFromUnexistingColumn(String tapSchemaName, ColumnHolder unexistingColumn) throws SQLException;
 
     void updateTapSchemaColumnValue(String tapSchemaName, String completeTableName, String columnName, String key, Object value) throws SQLException;
+
+    void alterDataType(String schemaName, String tableName, String columnName, String adqlDataType, Integer size) throws SQLException;
 }

TASMAN-core/src/main/java/it/inaf/ia2/tsm/datalayer/DBBrokerTemplate.java

@@ -781,17 +781,25 @@ public abstract class DBBrokerTemplate implements DBBroker {
     protected abstract Map<String, Map<String, Object>> getAllColumnsOriginalMetadata(String schemaName, String tableName) throws SQLException;
 
     @Override
-    public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, String tapSchemaVersion) throws SQLException {
+    public Map<String, Map<String, Object>> getAllColumnsMetadata(String schemaName, String tableName, TableModel tableModel, DataTypeMode dataTypeMode) throws SQLException {
 
         Map<String, Map<String, Object>> metadata = getAllColumnsOriginalMetadata(schemaName, tableName