Commit 4b9b9bba authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Consistency checking improvements

parent c6e34e6d
/*
* _____________________________________________________________________________
*
* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
* Trieste INAF - IA2 Italian Center for Astronomical Archives
* _____________________________________________________________________________
*
* Copyright (C) 2017 Istituto Nazionale di Astrofisica
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License Version 3 as published by the
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package it.inaf.ia2.tsm;
import java.io.Serializable;
import java.util.Objects;
import java.util.regex.Pattern;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class ColumnHolder implements Serializable {
private static final long serialVersionUID = 5837270912530393588L;
private String schemaName;
private String tableName;
private String columnName;
private ColumnHolder() {
}
public ColumnHolder(String schemaName, String tableName, String columnName) {
this.schemaName = schemaName;
this.tableName = tableName;
this.columnName = columnName;
}
public String getSchemaName() {
return schemaName;
}
public void setSchemaName(String schemaName) {
this.schemaName = schemaName;
}
public String getTableName() {
return tableName;
}
public void setTableName(String tableName) {
this.tableName = tableName;
}
public String getColumnName() {
return columnName;
}
public void setColumnName(String columnName) {
this.columnName = columnName;
}
@Override
public int hashCode() {
int hash = 7;
hash = 71 * hash + Objects.hashCode(this.schemaName);
hash = 71 * hash + Objects.hashCode(this.tableName);
hash = 71 * hash + Objects.hashCode(this.columnName);
return hash;
}
@Override
public boolean equals(Object obj) {
if (this == obj) {
return true;
}
if (obj == null) {
return false;
}
if (getClass() != obj.getClass()) {
return false;
}
final ColumnHolder other = (ColumnHolder) obj;
if (!Objects.equals(this.schemaName, other.schemaName)) {
return false;
}
if (!Objects.equals(this.tableName, other.tableName)) {
return false;
}
return Objects.equals(this.columnName, other.columnName);
}
@Override
public String toString() {
return String.format("%s.%s.%s", schemaName, tableName, columnName);
}
}
......@@ -22,24 +22,20 @@
*/
package it.inaf.ia2.tsm;
import it.inaf.ia2.tsm.datalayer.DatabaseType;
import it.inaf.ia2.tsm.datalayer.DBWrapper;
import it.inaf.ia2.tsm.model.ColumnModel;
import java.io.Serializable;
import java.sql.Connection;
import java.sql.PreparedStatement;
import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import javax.sql.DataSource;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
*
* TODO: Move queries into data layer classes.
* DataModel for consistency checking result.
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
......@@ -48,110 +44,35 @@ public class ConsistencyChecks implements Serializable {
private static final long serialVersionUID = 4412404312756740093L;
private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class);
public static class UnexistingColumn implements Serializable {
private static final long serialVersionUID = -4898369878807200093L;
private final String completeTableName;
private final String columnName;
private UnexistingColumn(String completeTableName, String columnName) {
this.completeTableName = completeTableName;
this.columnName = columnName;
}
public String getCompleteTableName() {
return completeTableName;
}
public String getColumnName() {
return columnName;
}
@Override
public String toString() {
return String.format("%s.%s", completeTableName, columnName);
}
}
public static class UnexistingKey implements Serializable {
private static final long serialVersionUID = 7891439129072900628L;
private final String keyId;
private final String fromTable;
private final String[] fromColumns;
private final String targetTable;
private final String[] targetColumns;
private UnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) {
this.keyId = keyId;
this.fromTable = fromTable;
this.fromColumns = fromColumns;
this.targetTable = targetTable;
this.targetColumns = targetColumns;
}
public String getKeyId() {
return keyId;
}
public String getFromTable() {
return fromTable;
}
public String[] getFromColumns() {
return fromColumns;
}
public String getTargetTable() {
return targetTable;
}
public String[] getTargetColumns() {
return targetColumns;
}
private String getColumnsString(String[] columns) {
StringBuilder sb = new StringBuilder();
boolean first = true;
for (String column : columns) {
if (!first) {
sb.append(",");
}
sb.append(column);
first = false;
}
return sb.toString();
}
@Override
public String toString() {
return String.format("[%s] %s(%s) -> %s(%s)", keyId,
fromTable, getColumnsString(fromColumns),
targetTable, getColumnsString(targetColumns));
}
}
private final List<InconsistentValue> inconsistencies;
private final List<InconsistentColumnProperty> inconsistencies;
private final Set<String> unexisingSchemas;
private final Set<String> unexisingTables;
private final List<UnexistingColumn> unexistingColumns;
private final List<UnexistingKey> unexistingKeys;
private final Set<ColumnHolder> unexistingColumns;
private final Set<KeyHolder> unexistingKeys;
private final Map<String, Set<String>> missingTables;
private final Map<String, Set<String>> tablesToAdd;
private final Map<ColumnHolder, ColumnModel> missingColumns;
private final Set<ColumnHolder> columnsToAdd;
private boolean missingObscore;
private boolean obscoreToAdd;
public ConsistencyChecks() {
inconsistencies = new ArrayList<>();
unexisingSchemas = new HashSet<>();
unexisingTables = new HashSet<>();
unexistingColumns = new ArrayList<>();
unexistingKeys = new ArrayList<>();
unexistingColumns = new HashSet<>();
unexistingKeys = new HashSet<>();
missingTables = new HashMap<>();
tablesToAdd = new HashMap<>();
missingColumns = new HashMap<>();
columnsToAdd = new HashSet<>();
}
public void addInconsistency(InconsistentValue problemDescription) {
public void addInconsistency(InconsistentColumnProperty problemDescription) {
inconsistencies.add(problemDescription);
}
public List<InconsistentValue> getInconsistencies() {
public List<InconsistentColumnProperty> getInconsistencies() {
return inconsistencies;
}
......@@ -171,192 +92,101 @@ public class ConsistencyChecks implements Serializable {
unexisingTables.add(schemaName + "." + tableSimpleName);
}
public List<UnexistingColumn> getUnexisingColumns() {
public Set<ColumnHolder> getUnexisingColumns() {
return unexistingColumns;
}
public void addUnexistingColumn(String completeTableName, String columnName) {
unexistingColumns.add(new UnexistingColumn(completeTableName, columnName));
public void addUnexistingColumn(String schemaName, String tableName, String columnName) {
unexistingColumns.add(new ColumnHolder(schemaName, tableName, columnName));
}
public void addUnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) {
if (keyId == null) {
throw new IllegalArgumentException("key_id can't be null");
}
unexistingKeys.add(new UnexistingKey(keyId, fromTable, fromColumns, targetTable, targetColumns));
unexistingKeys.add(new KeyHolder(keyId, fromTable, fromColumns, targetTable, targetColumns));
}
public List<UnexistingKey> getUnexistingKeys() {
public Set<KeyHolder> getUnexistingKeys() {
return unexistingKeys;
}
private Set<String> getKeysToRemove(Connection conn, String tapSchemaNameEscaped, DatabaseType dbType, String like) throws SQLException {
Set<String> ret = new HashSet<>();
String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, like + "%");
ps.setString(2, like + "%");
LOG.debug("Executing query: {} [{}]", query, like);
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
ret.add(rs.getString("key_id"));
}
}
}
return ret;
public void addMissingColumn(ColumnHolder columnHolder, ColumnModel columnModel) {
// Removing table from unexisting columns set
unexistingColumns.remove(columnHolder);
missingColumns.put(columnHolder, columnModel);
}
public boolean isInconsistent() {
return !inconsistencies.isEmpty() || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty();
public Map<ColumnHolder, ColumnModel> getMissingColumns() {
return missingColumns;
}
private void keysToRemoveFromUnexistingColumns(Connection conn, String tapSchemaNameEscaped, Set<String> keysToRemoveIds) throws SQLException {
for (UnexistingColumn unexistingColumn : unexistingColumns) {
StringBuilder sb = new StringBuilder();
sb.append("SELECT k.key_id AS key_id\n");
sb.append("FROM ");
sb.append(tapSchemaNameEscaped);
sb.append(".`keys` k\n");
sb.append("JOIN ");
sb.append(tapSchemaNameEscaped);
sb.append(".key_columns c ON k.key_id = c.key_id\n");
sb.append("WHERE (k.from_table = ? AND c.from_column = ?) OR (k.target_table = ? AND c.target_column = ?)");
String query = sb.toString();
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, unexistingColumn.getCompleteTableName());
ps.setString(2, unexistingColumn.getColumnName());
ps.setString(3, unexistingColumn.getCompleteTableName());
ps.setString(4, unexistingColumn.getColumnName());
LOG.debug("Executing query {}", query);
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
keysToRemoveIds.add(rs.getString("key_id"));
}
}
}
private void addTableToMap(String schemaName, String tableName, Map<String, Set<String>> map) {
Set<String> tables = map.get(schemaName);
if (tables == null) {
tables = new HashSet<>();
map.put(schemaName, tables);
}
tables.add(tableName);
}
public void amendTapSchema(DBWrapper dbWrapper, TapSchema tapSchema) throws SQLException {
Set<String> keysToRemoveIds = new HashSet<>();
public void addMissingTable(String schemaName, String tableName) {
// Removing table from unexisting table set
unexisingTables.remove(String.format("%s.%s", schemaName, tableName));
DatabaseType dbType = dbWrapper.getTapSchemaDatabaseType();
DataSource dataSource = dbWrapper.getTapSchemaDataSource();
String tapSchemaNameEscaped = TSMUtil.escapeName(tapSchema.getName(), dbType);
// Removing table from unexisting columns set
Iterator<ColumnHolder> ite = unexistingColumns.iterator();
while (ite.hasNext()) {
ColumnHolder ch = ite.next();
if (ch.getSchemaName().equals(schemaName) && ch.getTableName().equals(tableName)) {
ite.remove();
}
}
String query;
try (Connection conn = dataSource.getConnection()) {
addTableToMap(schemaName, tableName, missingTables);
}
for (String schema : unexisingSchemas) {
keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, schema));
}
for (String table : unexisingTables) {
keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, table));
}
public void addTableToAdd(String schemaName, String tableName) {
addTableToMap(schemaName, tableName, tablesToAdd);
}
keysToRemoveFromUnexistingColumns(conn, tapSchemaNameEscaped, keysToRemoveIds);
public Map<String, Set<String>> getTablesToAdd() {
return tablesToAdd;
}
for (UnexistingKey unexistingKey : unexistingKeys) {
keysToRemoveIds.add(unexistingKey.getKeyId());
}
public Map<String, Set<String>> getMissingTables() {
return missingTables;
}
conn.setAutoCommit(false);
LOG.debug("Starting transaction");
public void addColumnToAdd(ColumnHolder columnHolder) {
columnsToAdd.add(columnHolder);
}
try {
// Removing all key_columns
for (String keyId : keysToRemoveIds) {
query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("key_columns", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, keyId);
LOG.debug("Executing query {} [{}]", query, keyId);
ps.executeUpdate();
}
}
public Set<ColumnHolder> getColumnsToAdd() {
return columnsToAdd;
}
// Removing all keys
for (String keyId : keysToRemoveIds) {
query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, keyId);
LOG.debug("Executing query {} [{}]", query, keyId);
ps.executeUpdate();
}
}
public boolean isMissingObscore() {
return missingObscore;
}
// Removing all columns
for (UnexistingColumn unexistingColumn : unexistingColumns) {
query = String.format("DELETE FROM %s.%s WHERE table_name = ? AND column_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, unexistingColumn.getCompleteTableName());
ps.setString(2, unexistingColumn.getColumnName());
LOG.debug("Executing query {} [{}, {}]", query, unexistingColumn.getCompleteTableName(), unexistingColumn.getColumnName());
ps.executeUpdate();
}
}
for (String table : unexisingTables) {
query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, table);
LOG.debug("Executing query {} [{}]", query, table);
ps.executeUpdate();
}
}
for (String schema : unexisingSchemas) {
query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, schema + "%");
LOG.debug("Executing query {} [{}%]", query, schema);
ps.executeUpdate();
}
}
public void setMissingObscore(boolean missingObscore) {
this.missingObscore = missingObscore;
}
// Removing all tables
for (String table : unexisingTables) {
query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, table);
LOG.debug("Executing query {} [{}]", query, table);
ps.executeUpdate();
}
}
for (String schema : unexisingSchemas) {
query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, schema);
LOG.debug("Executing query {} [{}]", query, schema);
ps.executeUpdate();
}
}
public boolean isObscoreToAdd() {
return obscoreToAdd;
}
// Removing all schemas
for (String schema : unexisingSchemas) {
query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("schemas", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, schema);
LOG.debug("Executing query {} [{}]", query, schema);
ps.executeUpdate();
}
}
public void setObscoreToAdd(boolean obscoreToAdd) {
this.obscoreToAdd = obscoreToAdd;
}
conn.commit();
} catch (SQLException e) {
LOG.error("Exception detected. Executing rollback!", e);
try {
conn.rollback();
conn.setAutoCommit(true);
} catch (SQLException er) {
LOG.error("Exception during rollback", er);
throw er;
}
}
}
public boolean isInconsistent() {
return !inconsistencies.isEmpty()
|| !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty()
|| !missingTables.isEmpty() || !missingColumns.isEmpty()
|| !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty()
|| obscoreToAdd || missingObscore;
}
}
......@@ -64,6 +64,8 @@ public interface EntitiesContainer<T extends ChildEntity> {
*/
List<String> getAddableChildrenNames();
boolean isAddable(String childName);
/**
* Retrieve a list of children filtering them by a set of possible
* status.<br>
......
......@@ -22,40 +22,43 @@
*/
package it.inaf.ia2.tsm;
import java.io.Serializable;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class InconsistentValue {
private final String tapSchemaEntityType;
public class InconsistentColumnProperty implements Serializable {
private final String tapSchemaEntityDescription;
private static final long serialVersionUID = -5145865322582594970L;
private final String wrongPropertyName;
private String tableCompleteName;
private String columnName;
private String key;
private Object currentValue;
private Object correctValue;
private final Object currentValue;
private final Object correctValue;
private InconsistentColumnProperty() {
}
public InconsistentValue(String tapSchemaEntityType, String tapSchemaEntityDescription, String wrongPropertyName, Object currentValue, Object correctValue) {
this.tapSchemaEntityType = tapSchemaEntityType;
this.tapSchemaEntityDescription = tapSchemaEntityDescription;