Loading TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java 0 → 100644 +108 −0 Original line number Original line Diff line number Diff line /* * _____________________________________________________________________________ * * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of * Trieste INAF - IA2 Italian Center for Astronomical Archives * _____________________________________________________________________________ * * Copyright (C) 2017 Istituto Nazionale di Astrofisica * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License Version 3 as published by the * Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc., 51 * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package it.inaf.ia2.tsm; import java.io.Serializable; import java.util.Objects; import java.util.regex.Pattern; /** * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ public class ColumnHolder implements Serializable { private static final long serialVersionUID = 5837270912530393588L; private String schemaName; private String tableName; private String columnName; private ColumnHolder() { } public ColumnHolder(String schemaName, String tableName, String columnName) { this.schemaName = schemaName; this.tableName = tableName; this.columnName = columnName; } public String getSchemaName() { return schemaName; } public void setSchemaName(String schemaName) { this.schemaName = schemaName; } public String getTableName() { return tableName; } public void setTableName(String tableName) { this.tableName = tableName; } public String getColumnName() { return columnName; } public void setColumnName(String columnName) { this.columnName = columnName; } @Override public int hashCode() { int hash = 7; hash = 71 * hash + Objects.hashCode(this.schemaName); hash = 71 * hash + Objects.hashCode(this.tableName); hash = 71 * hash + Objects.hashCode(this.columnName); return hash; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } final ColumnHolder other = (ColumnHolder) obj; if (!Objects.equals(this.schemaName, other.schemaName)) { return false; } if (!Objects.equals(this.tableName, other.tableName)) { return false; } return Objects.equals(this.columnName, other.columnName); } @Override public String toString() { return String.format("%s.%s.%s", schemaName, tableName, columnName); } } TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java +85 −255 Original line number Original line Diff line number Diff line Loading @@ -22,24 +22,20 @@ */ */ package it.inaf.ia2.tsm; package it.inaf.ia2.tsm; import it.inaf.ia2.tsm.datalayer.DatabaseType; import it.inaf.ia2.tsm.model.ColumnModel; import it.inaf.ia2.tsm.datalayer.DBWrapper; import java.io.Serializable; import java.io.Serializable; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.List; import java.util.Map; import java.util.Set; import java.util.Set; import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory; /** /** * * DataModel for consistency checking result. * TODO: Move queries into data layer classes. * * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ */ Loading @@ -48,110 +44,35 @@ public class ConsistencyChecks implements Serializable { private static final long serialVersionUID = 4412404312756740093L; private static final long serialVersionUID = 4412404312756740093L; private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class); private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class); public static class UnexistingColumn implements Serializable { private final List<InconsistentColumnProperty> inconsistencies; private static final long serialVersionUID = -4898369878807200093L; private final String completeTableName; private final String columnName; private UnexistingColumn(String completeTableName, String columnName) { this.completeTableName = completeTableName; this.columnName = columnName; } public String getCompleteTableName() { return completeTableName; } public String getColumnName() { return columnName; } @Override public String toString() { return String.format("%s.%s", completeTableName, columnName); } } public static class UnexistingKey implements Serializable { private static final long serialVersionUID = 7891439129072900628L; private final String keyId; private final String fromTable; private final String[] fromColumns; private final String targetTable; private final String[] targetColumns; private UnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { this.keyId = keyId; this.fromTable = fromTable; this.fromColumns = fromColumns; this.targetTable = targetTable; this.targetColumns = targetColumns; } public String getKeyId() { return keyId; } public String getFromTable() { return fromTable; } public String[] getFromColumns() { return fromColumns; } public String getTargetTable() { return targetTable; } public String[] getTargetColumns() { return targetColumns; } private String getColumnsString(String[] columns) { StringBuilder sb = new StringBuilder(); boolean first = true; for (String column : columns) { if (!first) { sb.append(","); } sb.append(column); first = false; } return sb.toString(); } @Override public String toString() { return String.format("[%s] %s(%s) -> %s(%s)", keyId, fromTable, getColumnsString(fromColumns), targetTable, getColumnsString(targetColumns)); } } private final List<InconsistentValue> inconsistencies; private final Set<String> unexisingSchemas; private final Set<String> unexisingSchemas; private final Set<String> unexisingTables; private final Set<String> unexisingTables; private final List<UnexistingColumn> unexistingColumns; private final Set<ColumnHolder> unexistingColumns; private final List<UnexistingKey> unexistingKeys; private final Set<KeyHolder> unexistingKeys; private final Map<String, Set<String>> missingTables; private final Map<String, Set<String>> tablesToAdd; private final Map<ColumnHolder, ColumnModel> missingColumns; private final Set<ColumnHolder> columnsToAdd; private boolean missingObscore; private boolean obscoreToAdd; public ConsistencyChecks() { public ConsistencyChecks() { inconsistencies = new ArrayList<>(); inconsistencies = new ArrayList<>(); unexisingSchemas = new HashSet<>(); unexisingSchemas = new HashSet<>(); unexisingTables = new HashSet<>(); unexisingTables = new HashSet<>(); unexistingColumns = new ArrayList<>(); unexistingColumns = new HashSet<>(); unexistingKeys = new ArrayList<>(); unexistingKeys = new HashSet<>(); missingTables = new HashMap<>(); tablesToAdd = new HashMap<>(); missingColumns = new HashMap<>(); columnsToAdd = new HashSet<>(); } } public void addInconsistency(InconsistentValue problemDescription) { public void addInconsistency(InconsistentColumnProperty problemDescription) { inconsistencies.add(problemDescription); inconsistencies.add(problemDescription); } } public List<InconsistentValue> getInconsistencies() { public List<InconsistentColumnProperty> getInconsistencies() { return inconsistencies; return inconsistencies; } } Loading @@ -171,192 +92,101 @@ public class ConsistencyChecks implements Serializable { unexisingTables.add(schemaName + "." + tableSimpleName); unexisingTables.add(schemaName + "." + tableSimpleName); } } public List<UnexistingColumn> getUnexisingColumns() { public Set<ColumnHolder> getUnexisingColumns() { return unexistingColumns; return unexistingColumns; } } public void addUnexistingColumn(String completeTableName, String columnName) { public void addUnexistingColumn(String schemaName, String tableName, String columnName) { unexistingColumns.add(new UnexistingColumn(completeTableName, columnName)); unexistingColumns.add(new ColumnHolder(schemaName, tableName, columnName)); } } public void addUnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { public void addUnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { if (keyId == null) { if (keyId == null) { throw new IllegalArgumentException("key_id can't be null"); throw new IllegalArgumentException("key_id can't be null"); } } unexistingKeys.add(new UnexistingKey(keyId, fromTable, fromColumns, targetTable, targetColumns)); unexistingKeys.add(new KeyHolder(keyId, fromTable, fromColumns, targetTable, targetColumns)); } } public List<UnexistingKey> getUnexistingKeys() { public Set<KeyHolder> getUnexistingKeys() { return unexistingKeys; return unexistingKeys; } } private Set<String> getKeysToRemove(Connection conn, String tapSchemaNameEscaped, DatabaseType dbType, String like) throws SQLException { public void addMissingColumn(ColumnHolder columnHolder, ColumnModel columnModel) { Set<String> ret = new HashSet<>(); // Removing table from unexisting columns set String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType)); unexistingColumns.remove(columnHolder); try (PreparedStatement ps = conn.prepareStatement(query)) { missingColumns.put(columnHolder, columnModel); ps.setString(1, like + "%"); ps.setString(2, like + "%"); LOG.debug("Executing query: {} [{}]", query, like); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { ret.add(rs.getString("key_id")); } } } return ret; } } public boolean isInconsistent() { public Map<ColumnHolder, ColumnModel> getMissingColumns() { return !inconsistencies.isEmpty() || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty(); return missingColumns; } } private void keysToRemoveFromUnexistingColumns(Connection conn, String tapSchemaNameEscaped, Set<String> keysToRemoveIds) throws SQLException { private void addTableToMap(String schemaName, String tableName, Map<String, Set<String>> map) { for (UnexistingColumn unexistingColumn : unexistingColumns) { Set<String> tables = map.get(schemaName); if (tables == null) { StringBuilder sb = new StringBuilder(); tables = new HashSet<>(); sb.append("SELECT k.key_id AS key_id\n"); map.put(schemaName, tables); sb.append("FROM "); sb.append(tapSchemaNameEscaped); sb.append(".`keys` k\n"); sb.append("JOIN "); sb.append(tapSchemaNameEscaped); sb.append(".key_columns c ON k.key_id = c.key_id\n"); sb.append("WHERE (k.from_table = ? AND c.from_column = ?) OR (k.target_table = ? AND c.target_column = ?)"); String query = sb.toString(); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, unexistingColumn.getCompleteTableName()); ps.setString(2, unexistingColumn.getColumnName()); ps.setString(3, unexistingColumn.getCompleteTableName()); ps.setString(4, unexistingColumn.getColumnName()); LOG.debug("Executing query {}", query); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { keysToRemoveIds.add(rs.getString("key_id")); } } } } } tables.add(tableName); } } public void amendTapSchema(DBWrapper dbWrapper, TapSchema tapSchema) throws SQLException { public void addMissingTable(String schemaName, String tableName) { // Removing table from unexisting table set Set<String> keysToRemoveIds = new HashSet<>(); unexisingTables.remove(String.format("%s.%s", schemaName, tableName)); DatabaseType dbType = dbWrapper.getTapSchemaDatabaseType(); DataSource dataSource = dbWrapper.getTapSchemaDataSource(); String tapSchemaNameEscaped = TSMUtil.escapeName(tapSchema.getName(), dbType); String query; try (Connection conn = dataSource.getConnection()) { for (String schema : unexisingSchemas) { // Removing table from unexisting columns set keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, schema)); Iterator<ColumnHolder> ite = unexistingColumns.iterator(); while (ite.hasNext()) { ColumnHolder ch = ite.next(); if (ch.getSchemaName().equals(schemaName) && ch.getTableName().equals(tableName)) { ite.remove(); } } for (String table : unexisingTables) { keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, table)); } } keysToRemoveFromUnexistingColumns(conn, tapSchemaNameEscaped, keysToRemoveIds); addTableToMap(schemaName, tableName, missingTables); for (UnexistingKey unexistingKey : unexistingKeys) { keysToRemoveIds.add(unexistingKey.getKeyId()); } } conn.setAutoCommit(false); public void addTableToAdd(String schemaName, String tableName) { LOG.debug("Starting transaction"); addTableToMap(schemaName, tableName, tablesToAdd); try { // Removing all key_columns for (String keyId : keysToRemoveIds) { query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("key_columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, keyId); LOG.debug("Executing query {} [{}]", query, keyId); ps.executeUpdate(); } } } // Removing all keys public Map<String, Set<String>> getTablesToAdd() { for (String keyId : keysToRemoveIds) { return tablesToAdd; query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, keyId); LOG.debug("Executing query {} [{}]", query, keyId); ps.executeUpdate(); } } } // Removing all columns public Map<String, Set<String>> getMissingTables() { for (UnexistingColumn unexistingColumn : unexistingColumns) { return missingTables; query = String.format("DELETE FROM %s.%s WHERE table_name = ? AND column_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, unexistingColumn.getCompleteTableName()); ps.setString(2, unexistingColumn.getColumnName()); LOG.debug("Executing query {} [{}, {}]", query, unexistingColumn.getCompleteTableName(), unexistingColumn.getColumnName()); ps.executeUpdate(); } } for (String table : unexisingTables) { query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, table); LOG.debug("Executing query {} [{}]", query, table); ps.executeUpdate(); } } for (String schema : unexisingSchemas) { query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, schema + "%"); LOG.debug("Executing query {} [{}%]", query, schema); ps.executeUpdate(); } } } // Removing all tables public void addColumnToAdd(ColumnHolder columnHolder) { for (String table : unexisingTables) { columnsToAdd.add(columnHolder); query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, table); LOG.debug("Executing query {} [{}]", query, table); ps.executeUpdate(); } } for (String schema : unexisingSchemas) { query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, schema); LOG.debug("Executing query {} [{}]", query, schema); ps.executeUpdate(); } } } // Removing all schemas public Set<ColumnHolder> getColumnsToAdd() { for (String schema : unexisingSchemas) { return columnsToAdd; query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("schemas", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, schema); LOG.debug("Executing query {} [{}]", query, schema); ps.executeUpdate(); } } public boolean isMissingObscore() { return missingObscore; } } conn.commit(); public void setMissingObscore(boolean missingObscore) { } catch (SQLException e) { this.missingObscore = missingObscore; LOG.error("Exception detected. Executing rollback!", e); try { conn.rollback(); conn.setAutoCommit(true); } catch (SQLException er) { LOG.error("Exception during rollback", er); throw er; } } public boolean isObscoreToAdd() { return obscoreToAdd; } } public void setObscoreToAdd(boolean obscoreToAdd) { this.obscoreToAdd = obscoreToAdd; } } public boolean isInconsistent() { return !inconsistencies.isEmpty() || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty() || !missingTables.isEmpty() || !missingColumns.isEmpty() || !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty() || obscoreToAdd || missingObscore; } } } } TASMAN-core/src/main/java/it/inaf/ia2/tsm/EntitiesContainer.java +2 −0 Original line number Original line Diff line number Diff line Loading @@ -64,6 +64,8 @@ public interface EntitiesContainer<T extends ChildEntity> { */ */ List<String> getAddableChildrenNames(); List<String> getAddableChildrenNames(); boolean isAddable(String childName); /** /** * Retrieve a list of children filtering them by a set of possible * Retrieve a list of children filtering them by a set of possible * status.<br> * status.<br> Loading TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentValue.java→TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java +21 −18 Original line number Original line Diff line number Diff line Loading @@ -22,40 +22,43 @@ */ */ package it.inaf.ia2.tsm; package it.inaf.ia2.tsm; import java.io.Serializable; /** /** * * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ */ public class InconsistentValue { public class InconsistentColumnProperty implements Serializable { private final String tapSchemaEntityType; private final String tapSchemaEntityDescription; private static final long serialVersionUID = -5145865322582594970L; private final String wrongPropertyName; private String tableCompleteName; private String columnName; private String key; private Object currentValue; private Object correctValue; private final Object currentValue; private InconsistentColumnProperty() { } private final Object correctValue; public InconsistentValue(String tapSchemaEntityType, String tapSchemaEntityDescription, String wrongPropertyName, Object currentValue, Object correctValue) { public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) { this.tapSchemaEntityType = tapSchemaEntityType; this.tableCompleteName = tableCompleteName; this.tapSchemaEntityDescription = tapSchemaEntityDescription; this.columnName = columnName; this.wrongPropertyName = wrongPropertyName; this.key = key; this.currentValue = currentValue; this.currentValue = currentValue; this.correctValue = correctValue; this.correctValue = correctValue; } } public String getTapSchemaEntityType() { public String getTableCompleteName() { return tapSchemaEntityType; return tableCompleteName; } } public String getTapSchemaEntityDescription() { public String getColumnName() { return tapSchemaEntityDescription; return columnName; } } public String getWrongPropertyName() { public String getKey() { return wrongPropertyName; return key; } } public Object getCurrentValue() { public Object getCurrentValue() { Loading TASMAN-core/src/main/java/it/inaf/ia2/tsm/KeyHolder.java 0 → 100644 +91 −0 Original line number Original line Diff line number Diff line /* * _____________________________________________________________________________ * * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of * Trieste INAF - IA2 Italian Center for Astronomical Archives * _____________________________________________________________________________ * * Copyright (C) 2017 Istituto Nazionale di Astrofisica * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License Version 3 as published by the * Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc., 51 * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package it.inaf.ia2.tsm; import java.io.Serializable; /** * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ public class KeyHolder implements Serializable { private static final long serialVersionUID = 3711149068153684261L; private String keyId; private String fromTable; private String[] fromColumns; private String targetTable; private String[] targetColumns; private KeyHolder() { } public KeyHolder(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { this.keyId = keyId; this.fromTable = fromTable; this.fromColumns = fromColumns; this.targetTable = targetTable; this.targetColumns = targetColumns; } public String getKeyId() { return keyId; } public String getFromTable() { return fromTable; } public String[] getFromColumns() { return fromColumns; } public String getTargetTable() { return targetTable; } public String[] getTargetColumns() { return targetColumns; } private String getColumnsString(String[] columns) { StringBuilder sb = new StringBuilder(); boolean first = true; for (String column : columns) { if (!first) { sb.append(","); } sb.append(column); first = false; } return sb.toString(); } @Override public String toString() { return String.format("[%s] %s(%s) -> %s(%s)", keyId, fromTable, getColumnsString(fromColumns), targetTable, getColumnsString(targetColumns)); } } Loading
TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java 0 → 100644 +108 −0 Original line number Original line Diff line number Diff line /* * _____________________________________________________________________________ * * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of * Trieste INAF - IA2 Italian Center for Astronomical Archives * _____________________________________________________________________________ * * Copyright (C) 2017 Istituto Nazionale di Astrofisica * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License Version 3 as published by the * Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc., 51 * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package it.inaf.ia2.tsm; import java.io.Serializable; import java.util.Objects; import java.util.regex.Pattern; /** * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ public class ColumnHolder implements Serializable { private static final long serialVersionUID = 5837270912530393588L; private String schemaName; private String tableName; private String columnName; private ColumnHolder() { } public ColumnHolder(String schemaName, String tableName, String columnName) { this.schemaName = schemaName; this.tableName = tableName; this.columnName = columnName; } public String getSchemaName() { return schemaName; } public void setSchemaName(String schemaName) { this.schemaName = schemaName; } public String getTableName() { return tableName; } public void setTableName(String tableName) { this.tableName = tableName; } public String getColumnName() { return columnName; } public void setColumnName(String columnName) { this.columnName = columnName; } @Override public int hashCode() { int hash = 7; hash = 71 * hash + Objects.hashCode(this.schemaName); hash = 71 * hash + Objects.hashCode(this.tableName); hash = 71 * hash + Objects.hashCode(this.columnName); return hash; } @Override public boolean equals(Object obj) { if (this == obj) { return true; } if (obj == null) { return false; } if (getClass() != obj.getClass()) { return false; } final ColumnHolder other = (ColumnHolder) obj; if (!Objects.equals(this.schemaName, other.schemaName)) { return false; } if (!Objects.equals(this.tableName, other.tableName)) { return false; } return Objects.equals(this.columnName, other.columnName); } @Override public String toString() { return String.format("%s.%s.%s", schemaName, tableName, columnName); } }
TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java +85 −255 Original line number Original line Diff line number Diff line Loading @@ -22,24 +22,20 @@ */ */ package it.inaf.ia2.tsm; package it.inaf.ia2.tsm; import it.inaf.ia2.tsm.datalayer.DatabaseType; import it.inaf.ia2.tsm.model.ColumnModel; import it.inaf.ia2.tsm.datalayer.DBWrapper; import java.io.Serializable; import java.io.Serializable; import java.sql.Connection; import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.ArrayList; import java.util.HashMap; import java.util.HashSet; import java.util.HashSet; import java.util.Iterator; import java.util.List; import java.util.List; import java.util.Map; import java.util.Set; import java.util.Set; import javax.sql.DataSource; import org.slf4j.Logger; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.slf4j.LoggerFactory; /** /** * * DataModel for consistency checking result. * TODO: Move queries into data layer classes. * * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ */ Loading @@ -48,110 +44,35 @@ public class ConsistencyChecks implements Serializable { private static final long serialVersionUID = 4412404312756740093L; private static final long serialVersionUID = 4412404312756740093L; private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class); private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class); public static class UnexistingColumn implements Serializable { private final List<InconsistentColumnProperty> inconsistencies; private static final long serialVersionUID = -4898369878807200093L; private final String completeTableName; private final String columnName; private UnexistingColumn(String completeTableName, String columnName) { this.completeTableName = completeTableName; this.columnName = columnName; } public String getCompleteTableName() { return completeTableName; } public String getColumnName() { return columnName; } @Override public String toString() { return String.format("%s.%s", completeTableName, columnName); } } public static class UnexistingKey implements Serializable { private static final long serialVersionUID = 7891439129072900628L; private final String keyId; private final String fromTable; private final String[] fromColumns; private final String targetTable; private final String[] targetColumns; private UnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { this.keyId = keyId; this.fromTable = fromTable; this.fromColumns = fromColumns; this.targetTable = targetTable; this.targetColumns = targetColumns; } public String getKeyId() { return keyId; } public String getFromTable() { return fromTable; } public String[] getFromColumns() { return fromColumns; } public String getTargetTable() { return targetTable; } public String[] getTargetColumns() { return targetColumns; } private String getColumnsString(String[] columns) { StringBuilder sb = new StringBuilder(); boolean first = true; for (String column : columns) { if (!first) { sb.append(","); } sb.append(column); first = false; } return sb.toString(); } @Override public String toString() { return String.format("[%s] %s(%s) -> %s(%s)", keyId, fromTable, getColumnsString(fromColumns), targetTable, getColumnsString(targetColumns)); } } private final List<InconsistentValue> inconsistencies; private final Set<String> unexisingSchemas; private final Set<String> unexisingSchemas; private final Set<String> unexisingTables; private final Set<String> unexisingTables; private final List<UnexistingColumn> unexistingColumns; private final Set<ColumnHolder> unexistingColumns; private final List<UnexistingKey> unexistingKeys; private final Set<KeyHolder> unexistingKeys; private final Map<String, Set<String>> missingTables; private final Map<String, Set<String>> tablesToAdd; private final Map<ColumnHolder, ColumnModel> missingColumns; private final Set<ColumnHolder> columnsToAdd; private boolean missingObscore; private boolean obscoreToAdd; public ConsistencyChecks() { public ConsistencyChecks() { inconsistencies = new ArrayList<>(); inconsistencies = new ArrayList<>(); unexisingSchemas = new HashSet<>(); unexisingSchemas = new HashSet<>(); unexisingTables = new HashSet<>(); unexisingTables = new HashSet<>(); unexistingColumns = new ArrayList<>(); unexistingColumns = new HashSet<>(); unexistingKeys = new ArrayList<>(); unexistingKeys = new HashSet<>(); missingTables = new HashMap<>(); tablesToAdd = new HashMap<>(); missingColumns = new HashMap<>(); columnsToAdd = new HashSet<>(); } } public void addInconsistency(InconsistentValue problemDescription) { public void addInconsistency(InconsistentColumnProperty problemDescription) { inconsistencies.add(problemDescription); inconsistencies.add(problemDescription); } } public List<InconsistentValue> getInconsistencies() { public List<InconsistentColumnProperty> getInconsistencies() { return inconsistencies; return inconsistencies; } } Loading @@ -171,192 +92,101 @@ public class ConsistencyChecks implements Serializable { unexisingTables.add(schemaName + "." + tableSimpleName); unexisingTables.add(schemaName + "." + tableSimpleName); } } public List<UnexistingColumn> getUnexisingColumns() { public Set<ColumnHolder> getUnexisingColumns() { return unexistingColumns; return unexistingColumns; } } public void addUnexistingColumn(String completeTableName, String columnName) { public void addUnexistingColumn(String schemaName, String tableName, String columnName) { unexistingColumns.add(new UnexistingColumn(completeTableName, columnName)); unexistingColumns.add(new ColumnHolder(schemaName, tableName, columnName)); } } public void addUnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { public void addUnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { if (keyId == null) { if (keyId == null) { throw new IllegalArgumentException("key_id can't be null"); throw new IllegalArgumentException("key_id can't be null"); } } unexistingKeys.add(new UnexistingKey(keyId, fromTable, fromColumns, targetTable, targetColumns)); unexistingKeys.add(new KeyHolder(keyId, fromTable, fromColumns, targetTable, targetColumns)); } } public List<UnexistingKey> getUnexistingKeys() { public Set<KeyHolder> getUnexistingKeys() { return unexistingKeys; return unexistingKeys; } } private Set<String> getKeysToRemove(Connection conn, String tapSchemaNameEscaped, DatabaseType dbType, String like) throws SQLException { public void addMissingColumn(ColumnHolder columnHolder, ColumnModel columnModel) { Set<String> ret = new HashSet<>(); // Removing table from unexisting columns set String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType)); unexistingColumns.remove(columnHolder); try (PreparedStatement ps = conn.prepareStatement(query)) { missingColumns.put(columnHolder, columnModel); ps.setString(1, like + "%"); ps.setString(2, like + "%"); LOG.debug("Executing query: {} [{}]", query, like); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { ret.add(rs.getString("key_id")); } } } return ret; } } public boolean isInconsistent() { public Map<ColumnHolder, ColumnModel> getMissingColumns() { return !inconsistencies.isEmpty() || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty(); return missingColumns; } } private void keysToRemoveFromUnexistingColumns(Connection conn, String tapSchemaNameEscaped, Set<String> keysToRemoveIds) throws SQLException { private void addTableToMap(String schemaName, String tableName, Map<String, Set<String>> map) { for (UnexistingColumn unexistingColumn : unexistingColumns) { Set<String> tables = map.get(schemaName); if (tables == null) { StringBuilder sb = new StringBuilder(); tables = new HashSet<>(); sb.append("SELECT k.key_id AS key_id\n"); map.put(schemaName, tables); sb.append("FROM "); sb.append(tapSchemaNameEscaped); sb.append(".`keys` k\n"); sb.append("JOIN "); sb.append(tapSchemaNameEscaped); sb.append(".key_columns c ON k.key_id = c.key_id\n"); sb.append("WHERE (k.from_table = ? AND c.from_column = ?) OR (k.target_table = ? AND c.target_column = ?)"); String query = sb.toString(); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, unexistingColumn.getCompleteTableName()); ps.setString(2, unexistingColumn.getColumnName()); ps.setString(3, unexistingColumn.getCompleteTableName()); ps.setString(4, unexistingColumn.getColumnName()); LOG.debug("Executing query {}", query); try (ResultSet rs = ps.executeQuery()) { while (rs.next()) { keysToRemoveIds.add(rs.getString("key_id")); } } } } } tables.add(tableName); } } public void amendTapSchema(DBWrapper dbWrapper, TapSchema tapSchema) throws SQLException { public void addMissingTable(String schemaName, String tableName) { // Removing table from unexisting table set Set<String> keysToRemoveIds = new HashSet<>(); unexisingTables.remove(String.format("%s.%s", schemaName, tableName)); DatabaseType dbType = dbWrapper.getTapSchemaDatabaseType(); DataSource dataSource = dbWrapper.getTapSchemaDataSource(); String tapSchemaNameEscaped = TSMUtil.escapeName(tapSchema.getName(), dbType); String query; try (Connection conn = dataSource.getConnection()) { for (String schema : unexisingSchemas) { // Removing table from unexisting columns set keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, schema)); Iterator<ColumnHolder> ite = unexistingColumns.iterator(); while (ite.hasNext()) { ColumnHolder ch = ite.next(); if (ch.getSchemaName().equals(schemaName) && ch.getTableName().equals(tableName)) { ite.remove(); } } for (String table : unexisingTables) { keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, table)); } } keysToRemoveFromUnexistingColumns(conn, tapSchemaNameEscaped, keysToRemoveIds); addTableToMap(schemaName, tableName, missingTables); for (UnexistingKey unexistingKey : unexistingKeys) { keysToRemoveIds.add(unexistingKey.getKeyId()); } } conn.setAutoCommit(false); public void addTableToAdd(String schemaName, String tableName) { LOG.debug("Starting transaction"); addTableToMap(schemaName, tableName, tablesToAdd); try { // Removing all key_columns for (String keyId : keysToRemoveIds) { query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("key_columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, keyId); LOG.debug("Executing query {} [{}]", query, keyId); ps.executeUpdate(); } } } // Removing all keys public Map<String, Set<String>> getTablesToAdd() { for (String keyId : keysToRemoveIds) { return tablesToAdd; query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, keyId); LOG.debug("Executing query {} [{}]", query, keyId); ps.executeUpdate(); } } } // Removing all columns public Map<String, Set<String>> getMissingTables() { for (UnexistingColumn unexistingColumn : unexistingColumns) { return missingTables; query = String.format("DELETE FROM %s.%s WHERE table_name = ? AND column_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, unexistingColumn.getCompleteTableName()); ps.setString(2, unexistingColumn.getColumnName()); LOG.debug("Executing query {} [{}, {}]", query, unexistingColumn.getCompleteTableName(), unexistingColumn.getColumnName()); ps.executeUpdate(); } } for (String table : unexisingTables) { query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, table); LOG.debug("Executing query {} [{}]", query, table); ps.executeUpdate(); } } for (String schema : unexisingSchemas) { query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, schema + "%"); LOG.debug("Executing query {} [{}%]", query, schema); ps.executeUpdate(); } } } // Removing all tables public void addColumnToAdd(ColumnHolder columnHolder) { for (String table : unexisingTables) { columnsToAdd.add(columnHolder); query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, table); LOG.debug("Executing query {} [{}]", query, table); ps.executeUpdate(); } } for (String schema : unexisingSchemas) { query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, schema); LOG.debug("Executing query {} [{}]", query, schema); ps.executeUpdate(); } } } // Removing all schemas public Set<ColumnHolder> getColumnsToAdd() { for (String schema : unexisingSchemas) { return columnsToAdd; query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("schemas", dbType)); try (PreparedStatement ps = conn.prepareStatement(query)) { ps.setString(1, schema); LOG.debug("Executing query {} [{}]", query, schema); ps.executeUpdate(); } } public boolean isMissingObscore() { return missingObscore; } } conn.commit(); public void setMissingObscore(boolean missingObscore) { } catch (SQLException e) { this.missingObscore = missingObscore; LOG.error("Exception detected. Executing rollback!", e); try { conn.rollback(); conn.setAutoCommit(true); } catch (SQLException er) { LOG.error("Exception during rollback", er); throw er; } } public boolean isObscoreToAdd() { return obscoreToAdd; } } public void setObscoreToAdd(boolean obscoreToAdd) { this.obscoreToAdd = obscoreToAdd; } } public boolean isInconsistent() { return !inconsistencies.isEmpty() || !unexisingSchemas.isEmpty() || !unexisingTables.isEmpty() || !unexistingColumns.isEmpty() || !missingTables.isEmpty() || !missingColumns.isEmpty() || !columnsToAdd.isEmpty() || !tablesToAdd.isEmpty() || obscoreToAdd || missingObscore; } } } }
TASMAN-core/src/main/java/it/inaf/ia2/tsm/EntitiesContainer.java +2 −0 Original line number Original line Diff line number Diff line Loading @@ -64,6 +64,8 @@ public interface EntitiesContainer<T extends ChildEntity> { */ */ List<String> getAddableChildrenNames(); List<String> getAddableChildrenNames(); boolean isAddable(String childName); /** /** * Retrieve a list of children filtering them by a set of possible * Retrieve a list of children filtering them by a set of possible * status.<br> * status.<br> Loading
TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentValue.java→TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java +21 −18 Original line number Original line Diff line number Diff line Loading @@ -22,40 +22,43 @@ */ */ package it.inaf.ia2.tsm; package it.inaf.ia2.tsm; import java.io.Serializable; /** /** * * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ */ public class InconsistentValue { public class InconsistentColumnProperty implements Serializable { private final String tapSchemaEntityType; private final String tapSchemaEntityDescription; private static final long serialVersionUID = -5145865322582594970L; private final String wrongPropertyName; private String tableCompleteName; private String columnName; private String key; private Object currentValue; private Object correctValue; private final Object currentValue; private InconsistentColumnProperty() { } private final Object correctValue; public InconsistentValue(String tapSchemaEntityType, String tapSchemaEntityDescription, String wrongPropertyName, Object currentValue, Object correctValue) { public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) { this.tapSchemaEntityType = tapSchemaEntityType; this.tableCompleteName = tableCompleteName; this.tapSchemaEntityDescription = tapSchemaEntityDescription; this.columnName = columnName; this.wrongPropertyName = wrongPropertyName; this.key = key; this.currentValue = currentValue; this.currentValue = currentValue; this.correctValue = correctValue; this.correctValue = correctValue; } } public String getTapSchemaEntityType() { public String getTableCompleteName() { return tapSchemaEntityType; return tableCompleteName; } } public String getTapSchemaEntityDescription() { public String getColumnName() { return tapSchemaEntityDescription; return columnName; } } public String getWrongPropertyName() { public String getKey() { return wrongPropertyName; return key; } } public Object getCurrentValue() { public Object getCurrentValue() { Loading
TASMAN-core/src/main/java/it/inaf/ia2/tsm/KeyHolder.java 0 → 100644 +91 −0 Original line number Original line Diff line number Diff line /* * _____________________________________________________________________________ * * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of * Trieste INAF - IA2 Italian Center for Astronomical Archives * _____________________________________________________________________________ * * Copyright (C) 2017 Istituto Nazionale di Astrofisica * * This program is free software; you can redistribute it and/or modify it under * the terms of the GNU General Public License Version 3 as published by the * Free Software Foundation. * * This program is distributed in the hope that it will be useful, but WITHOUT * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more * details. * * You should have received a copy of the GNU General Public License along with * this program; if not, write to the Free Software Foundation, Inc., 51 * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA. */ package it.inaf.ia2.tsm; import java.io.Serializable; /** * * @author Sonia Zorba {@literal <zorba at oats.inaf.it>} */ public class KeyHolder implements Serializable { private static final long serialVersionUID = 3711149068153684261L; private String keyId; private String fromTable; private String[] fromColumns; private String targetTable; private String[] targetColumns; private KeyHolder() { } public KeyHolder(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) { this.keyId = keyId; this.fromTable = fromTable; this.fromColumns = fromColumns; this.targetTable = targetTable; this.targetColumns = targetColumns; } public String getKeyId() { return keyId; } public String getFromTable() { return fromTable; } public String[] getFromColumns() { return fromColumns; } public String getTargetTable() { return targetTable; } public String[] getTargetColumns() { return targetColumns; } private String getColumnsString(String[] columns) { StringBuilder sb = new StringBuilder(); boolean first = true; for (String column : columns) { if (!first) { sb.append(","); } sb.append(column); first = false; } return sb.toString(); } @Override public String toString() { return String.format("[%s] %s(%s) -> %s(%s)", keyId, fromTable, getColumnsString(fromColumns), targetTable, getColumnsString(targetColumns)); } }