Consistency checking improvements (4b9b9bba) · Commits · Marco Molinaro / tap_schema_manager

TASMAN-core/src/main/java/it/inaf/ia2/tsm/ColumnHolder.java

0 → 100644

+108 −0

Original line number	Diff line number	Diff line
		/*
		* _____________________________________________________________________________
		*
		* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
		* Trieste INAF - IA2 Italian Center for Astronomical Archives
		* _____________________________________________________________________________
		*
		* Copyright (C) 2017 Istituto Nazionale di Astrofisica
		*
		* This program is free software; you can redistribute it and/or modify it under
		* the terms of the GNU General Public License Version 3 as published by the
		* Free Software Foundation.
		*
		* This program is distributed in the hope that it will be useful, but WITHOUT
		* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		* details.
		*
		* You should have received a copy of the GNU General Public License along with
		* this program; if not, write to the Free Software Foundation, Inc., 51
		* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
		*/
		package it.inaf.ia2.tsm;

		import java.io.Serializable;
		import java.util.Objects;
		import java.util.regex.Pattern;

		/**
		*
		* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
		*/
		public class ColumnHolder implements Serializable {

		private static final long serialVersionUID = 5837270912530393588L;

		private String schemaName;
		private String tableName;
		private String columnName;

		private ColumnHolder() {
		}

		public ColumnHolder(String schemaName, String tableName, String columnName) {
		this.schemaName = schemaName;
		this.tableName = tableName;
		this.columnName = columnName;
		}

		public String getSchemaName() {
		return schemaName;
		}

		public void setSchemaName(String schemaName) {
		this.schemaName = schemaName;
		}

		public String getTableName() {
		return tableName;
		}

		public void setTableName(String tableName) {
		this.tableName = tableName;
		}

		public String getColumnName() {
		return columnName;
		}

		public void setColumnName(String columnName) {
		this.columnName = columnName;
		}

		@Override
		public int hashCode() {
		int hash = 7;
		hash = 71 * hash + Objects.hashCode(this.schemaName);
		hash = 71 * hash + Objects.hashCode(this.tableName);
		hash = 71 * hash + Objects.hashCode(this.columnName);
		return hash;
		}

		@Override
		public boolean equals(Object obj) {
		if (this == obj) {
		return true;
		}
		if (obj == null) {
		return false;
		}
		if (getClass() != obj.getClass()) {
		return false;
		}
		final ColumnHolder other = (ColumnHolder) obj;
		if (!Objects.equals(this.schemaName, other.schemaName)) {
		return false;
		}
		if (!Objects.equals(this.tableName, other.tableName)) {
		return false;
		}
		return Objects.equals(this.columnName, other.columnName);
		}

		@Override
		public String toString() {
		return String.format("%s.%s.%s", schemaName, tableName, columnName);
		}
		}

TASMAN-core/src/main/java/it/inaf/ia2/tsm/ConsistencyChecks.java

+85 −255

Original line number	Diff line number	Diff line
		@@ -22,24 +22,20 @@
		*/
		package it.inaf.ia2.tsm;

		import it.inaf.ia2.tsm.datalayer.DatabaseType;
		import it.inaf.ia2.tsm.datalayer.DBWrapper;
		import it.inaf.ia2.tsm.model.ColumnModel;
		import java.io.Serializable;
		import java.sql.Connection;
		import java.sql.PreparedStatement;
		import java.sql.ResultSet;
		import java.sql.SQLException;
		import java.util.ArrayList;
		import java.util.HashMap;
		import java.util.HashSet;
		import java.util.Iterator;
		import java.util.List;
		import java.util.Map;
		import java.util.Set;
		import javax.sql.DataSource;
		import org.slf4j.Logger;
		import org.slf4j.LoggerFactory;

		/**
		*
		* TODO: Move queries into data layer classes.
		* DataModel for consistency checking result.
		*
		* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
		*/
		@@ -48,110 +44,35 @@ public class ConsistencyChecks implements Serializable {
		private static final long serialVersionUID = 4412404312756740093L;
		private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class);

		public static class UnexistingColumn implements Serializable {

		private static final long serialVersionUID = -4898369878807200093L;

		private final String completeTableName;
		private final String columnName;

		private UnexistingColumn(String completeTableName, String columnName) {
		this.completeTableName = completeTableName;
		this.columnName = columnName;
		}

		public String getCompleteTableName() {
		return completeTableName;
		}

		public String getColumnName() {
		return columnName;
		}

		@Override
		public String toString() {
		return String.format("%s.%s", completeTableName, columnName);
		}
		}

		public static class UnexistingKey implements Serializable {

		private static final long serialVersionUID = 7891439129072900628L;

		private final String keyId;
		private final String fromTable;
		private final String[] fromColumns;
		private final String targetTable;
		private final String[] targetColumns;

		private UnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) {
		this.keyId = keyId;
		this.fromTable = fromTable;
		this.fromColumns = fromColumns;
		this.targetTable = targetTable;
		this.targetColumns = targetColumns;
		}

		public String getKeyId() {
		return keyId;
		}

		public String getFromTable() {
		return fromTable;
		}

		public String[] getFromColumns() {
		return fromColumns;
		}

		public String getTargetTable() {
		return targetTable;
		}

		public String[] getTargetColumns() {
		return targetColumns;
		}

		private String getColumnsString(String[] columns) {
		StringBuilder sb = new StringBuilder();
		boolean first = true;
		for (String column : columns) {
		if (!first) {
		sb.append(",");
		}
		sb.append(column);
		first = false;
		}
		return sb.toString();
		}

		@Override
		public String toString() {
		return String.format("[%s] %s(%s) -> %s(%s)", keyId,
		fromTable, getColumnsString(fromColumns),
		targetTable, getColumnsString(targetColumns));
		}
		}

		private final List<InconsistentValue> inconsistencies;
		private final List<InconsistentColumnProperty> inconsistencies;
		private final Set<String> unexisingSchemas;
		private final Set<String> unexisingTables;
		private final List<UnexistingColumn> unexistingColumns;
		private final List<UnexistingKey> unexistingKeys;
		private final Set<ColumnHolder> unexistingColumns;
		private final Set<KeyHolder> unexistingKeys;
		private final Map<String, Set<String>> missingTables;
		private final Map<String, Set<String>> tablesToAdd;
		private final Map<ColumnHolder, ColumnModel> missingColumns;
		private final Set<ColumnHolder> columnsToAdd;
		private boolean missingObscore;
		private boolean obscoreToAdd;

		public ConsistencyChecks() {
		inconsistencies = new ArrayList<>();
		unexisingSchemas = new HashSet<>();
		unexisingTables = new HashSet<>();
		unexistingColumns = new ArrayList<>();
		unexistingKeys = new ArrayList<>();
		unexistingColumns = new HashSet<>();
		unexistingKeys = new HashSet<>();
		missingTables = new HashMap<>();
		tablesToAdd = new HashMap<>();
		missingColumns = new HashMap<>();
		columnsToAdd = new HashSet<>();
		}

		public void addInconsistency(InconsistentValue problemDescription) {
		public void addInconsistency(InconsistentColumnProperty problemDescription) {
		inconsistencies.add(problemDescription);
		}

		public List<InconsistentValue> getInconsistencies() {
		public List<InconsistentColumnProperty> getInconsistencies() {
		return inconsistencies;
		}

		@@ -171,192 +92,101 @@ public class ConsistencyChecks implements Serializable {
		unexisingTables.add(schemaName + "." + tableSimpleName);
		}

		public List<UnexistingColumn> getUnexisingColumns() {
		public Set<ColumnHolder> getUnexisingColumns() {
		return unexistingColumns;
		}

		public void addUnexistingColumn(String completeTableName, String columnName) {
		unexistingColumns.add(new UnexistingColumn(completeTableName, columnName));
		public void addUnexistingColumn(String schemaName, String tableName, String columnName) {
		unexistingColumns.add(new ColumnHolder(schemaName, tableName, columnName));
		}

		public void addUnexistingKey(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) {
		if (keyId == null) {
		throw new IllegalArgumentException("key_id can't be null");
		}
		unexistingKeys.add(new UnexistingKey(keyId, fromTable, fromColumns, targetTable, targetColumns));
		unexistingKeys.add(new KeyHolder(keyId, fromTable, fromColumns, targetTable, targetColumns));
		}

		public List<UnexistingKey> getUnexistingKeys() {
		public Set<KeyHolder> getUnexistingKeys() {
		return unexistingKeys;
		}

		private Set<String> getKeysToRemove(Connection conn, String tapSchemaNameEscaped, DatabaseType dbType, String like) throws SQLException {
		Set<String> ret = new HashSet<>();
		String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, like + "%");
		ps.setString(2, like + "%");
		LOG.debug("Executing query: {} [{}]", query, like);
		try (ResultSet rs = ps.executeQuery()) {
		while (rs.next()) {
		ret.add(rs.getString("key_id"));
		}
		}
		}
		return ret;
		public void addMissingColumn(ColumnHolder columnHolder, ColumnModel columnModel) {
		// Removing table from unexisting columns set
		unexistingColumns.remove(columnHolder);
		missingColumns.put(columnHolder, columnModel);
		}

		public boolean isInconsistent() {
		return !inconsistencies.isEmpty() \|\| !unexisingSchemas.isEmpty() \|\| !unexisingTables.isEmpty() \|\| !unexistingColumns.isEmpty();
		public Map<ColumnHolder, ColumnModel> getMissingColumns() {
		return missingColumns;
		}

		private void keysToRemoveFromUnexistingColumns(Connection conn, String tapSchemaNameEscaped, Set<String> keysToRemoveIds) throws SQLException {
		for (UnexistingColumn unexistingColumn : unexistingColumns) {

		StringBuilder sb = new StringBuilder();
		sb.append("SELECT k.key_id AS key_id\n");
		sb.append("FROM ");
		sb.append(tapSchemaNameEscaped);
		sb.append(".`keys` k\n");
		sb.append("JOIN ");
		sb.append(tapSchemaNameEscaped);
		sb.append(".key_columns c ON k.key_id = c.key_id\n");
		sb.append("WHERE (k.from_table = ? AND c.from_column = ?) OR (k.target_table = ? AND c.target_column = ?)");

		String query = sb.toString();

		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, unexistingColumn.getCompleteTableName());
		ps.setString(2, unexistingColumn.getColumnName());
		ps.setString(3, unexistingColumn.getCompleteTableName());
		ps.setString(4, unexistingColumn.getColumnName());

		LOG.debug("Executing query {}", query);

		try (ResultSet rs = ps.executeQuery()) {
		while (rs.next()) {
		keysToRemoveIds.add(rs.getString("key_id"));
		}
		}
		}
		private void addTableToMap(String schemaName, String tableName, Map<String, Set<String>> map) {
		Set<String> tables = map.get(schemaName);
		if (tables == null) {
		tables = new HashSet<>();
		map.put(schemaName, tables);
		}
		tables.add(tableName);
		}

		public void amendTapSchema(DBWrapper dbWrapper, TapSchema tapSchema) throws SQLException {

		Set<String> keysToRemoveIds = new HashSet<>();

		DatabaseType dbType = dbWrapper.getTapSchemaDatabaseType();
		DataSource dataSource = dbWrapper.getTapSchemaDataSource();

		String tapSchemaNameEscaped = TSMUtil.escapeName(tapSchema.getName(), dbType);

		String query;
		try (Connection conn = dataSource.getConnection()) {
		public void addMissingTable(String schemaName, String tableName) {
		// Removing table from unexisting table set
		unexisingTables.remove(String.format("%s.%s", schemaName, tableName));

		for (String schema : unexisingSchemas) {
		keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, schema));
		// Removing table from unexisting columns set
		Iterator<ColumnHolder> ite = unexistingColumns.iterator();
		while (ite.hasNext()) {
		ColumnHolder ch = ite.next();
		if (ch.getSchemaName().equals(schemaName) && ch.getTableName().equals(tableName)) {
		ite.remove();
		}
		for (String table : unexisingTables) {
		keysToRemoveIds.addAll(getKeysToRemove(conn, tapSchemaNameEscaped, dbType, table));
		}

		keysToRemoveFromUnexistingColumns(conn, tapSchemaNameEscaped, keysToRemoveIds);

		for (UnexistingKey unexistingKey : unexistingKeys) {
		keysToRemoveIds.add(unexistingKey.getKeyId());
		addTableToMap(schemaName, tableName, missingTables);
		}

		conn.setAutoCommit(false);
		LOG.debug("Starting transaction");

		try {
		// Removing all key_columns
		for (String keyId : keysToRemoveIds) {
		query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("key_columns", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, keyId);
		LOG.debug("Executing query {} [{}]", query, keyId);
		ps.executeUpdate();
		}
		public void addTableToAdd(String schemaName, String tableName) {
		addTableToMap(schemaName, tableName, tablesToAdd);
		}

		// Removing all keys
		for (String keyId : keysToRemoveIds) {
		query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, keyId);
		LOG.debug("Executing query {} [{}]", query, keyId);
		ps.executeUpdate();
		}
		public Map<String, Set<String>> getTablesToAdd() {
		return tablesToAdd;
		}

		// Removing all columns
		for (UnexistingColumn unexistingColumn : unexistingColumns) {
		query = String.format("DELETE FROM %s.%s WHERE table_name = ? AND column_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, unexistingColumn.getCompleteTableName());
		ps.setString(2, unexistingColumn.getColumnName());
		LOG.debug("Executing query {} [{}, {}]", query, unexistingColumn.getCompleteTableName(), unexistingColumn.getColumnName());
		ps.executeUpdate();
		}
		}
		for (String table : unexisingTables) {
		query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, table);
		LOG.debug("Executing query {} [{}]", query, table);
		ps.executeUpdate();
		}
		}
		for (String schema : unexisingSchemas) {
		query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, schema + "%");
		LOG.debug("Executing query {} [{}%]", query, schema);
		ps.executeUpdate();
		}
		public Map<String, Set<String>> getMissingTables() {
		return missingTables;
		}

		// Removing all tables
		for (String table : unexisingTables) {
		query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, table);
		LOG.debug("Executing query {} [{}]", query, table);
		ps.executeUpdate();
		}
		}
		for (String schema : unexisingSchemas) {
		query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, schema);
		LOG.debug("Executing query {} [{}]", query, schema);
		ps.executeUpdate();
		}
		public void addColumnToAdd(ColumnHolder columnHolder) {
		columnsToAdd.add(columnHolder);
		}

		// Removing all schemas
		for (String schema : unexisingSchemas) {
		query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("schemas", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, schema);
		LOG.debug("Executing query {} [{}]", query, schema);
		ps.executeUpdate();
		public Set<ColumnHolder> getColumnsToAdd() {
		return columnsToAdd;
		}

		public boolean isMissingObscore() {
		return missingObscore;
		}

		conn.commit();
		} catch (SQLException e) {
		LOG.error("Exception detected. Executing rollback!", e);
		try {
		conn.rollback();
		conn.setAutoCommit(true);
		} catch (SQLException er) {
		LOG.error("Exception during rollback", er);
		throw er;
		public void setMissingObscore(boolean missingObscore) {
		this.missingObscore = missingObscore;
		}

		public boolean isObscoreToAdd() {
		return obscoreToAdd;
		}

		public void setObscoreToAdd(boolean obscoreToAdd) {
		this.obscoreToAdd = obscoreToAdd;
		}

		public boolean isInconsistent() {
		return !inconsistencies.isEmpty()
		\|\| !unexisingSchemas.isEmpty() \|\| !unexisingTables.isEmpty() \|\| !unexistingColumns.isEmpty()
		\|\| !missingTables.isEmpty() \|\| !missingColumns.isEmpty()
		\|\| !columnsToAdd.isEmpty() \|\| !tablesToAdd.isEmpty()
		\|\| obscoreToAdd \|\| missingObscore;
		}
		}

TASMAN-core/src/main/java/it/inaf/ia2/tsm/EntitiesContainer.java

+2 −0

Original line number	Diff line number	Diff line
		@@ -64,6 +64,8 @@ public interface EntitiesContainer<T extends ChildEntity> {
		*/
		List<String> getAddableChildrenNames();

		boolean isAddable(String childName);

		/**
		* Retrieve a list of children filtering them by a set of possible
		* status.<br>

TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentValue.java→TASMAN-core/src/main/java/it/inaf/ia2/tsm/InconsistentColumnProperty.java

+21 −18

Original line number	Diff line number	Diff line
		@@ -22,40 +22,43 @@
		*/
		package it.inaf.ia2.tsm;

		import java.io.Serializable;

		/**
		*
		* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
		*/
		public class InconsistentValue {

		private final String tapSchemaEntityType;
		public class InconsistentColumnProperty implements Serializable {

		private final String tapSchemaEntityDescription;
		private static final long serialVersionUID = -5145865322582594970L;

		private final String wrongPropertyName;
		private String tableCompleteName;
		private String columnName;
		private String key;
		private Object currentValue;
		private Object correctValue;

		private final Object currentValue;

		private final Object correctValue;
		private InconsistentColumnProperty() {
		}

		public InconsistentValue(String tapSchemaEntityType, String tapSchemaEntityDescription, String wrongPropertyName, Object currentValue, Object correctValue) {
		this.tapSchemaEntityType = tapSchemaEntityType;
		this.tapSchemaEntityDescription = tapSchemaEntityDescription;
		this.wrongPropertyName = wrongPropertyName;
		public InconsistentColumnProperty(String tableCompleteName, String columnName, String key, Object currentValue, Object correctValue) {
		this.tableCompleteName = tableCompleteName;
		this.columnName = columnName;
		this.key = key;
		this.currentValue = currentValue;
		this.correctValue = correctValue;
		}

		public String getTapSchemaEntityType() {
		return tapSchemaEntityType;
		public String getTableCompleteName() {
		return tableCompleteName;
		}

		public String getTapSchemaEntityDescription() {
		return tapSchemaEntityDescription;
		public String getColumnName() {
		return columnName;
		}

		public String getWrongPropertyName() {
		return wrongPropertyName;
		public String getKey() {
		return key;
		}

		public Object getCurrentValue() {

TASMAN-core/src/main/java/it/inaf/ia2/tsm/KeyHolder.java

0 → 100644

+91 −0

Original line number	Diff line number	Diff line
		/*
		* _____________________________________________________________________________
		*
		* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
		* Trieste INAF - IA2 Italian Center for Astronomical Archives
		* _____________________________________________________________________________
		*
		* Copyright (C) 2017 Istituto Nazionale di Astrofisica
		*
		* This program is free software; you can redistribute it and/or modify it under
		* the terms of the GNU General Public License Version 3 as published by the
		* Free Software Foundation.
		*
		* This program is distributed in the hope that it will be useful, but WITHOUT
		* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		* details.
		*
		* You should have received a copy of the GNU General Public License along with
		* this program; if not, write to the Free Software Foundation, Inc., 51
		* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
		*/
		package it.inaf.ia2.tsm;

		import java.io.Serializable;

		/**
		*
		* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
		*/
		public class KeyHolder implements Serializable {

		private static final long serialVersionUID = 3711149068153684261L;

		private String keyId;
		private String fromTable;
		private String[] fromColumns;
		private String targetTable;
		private String[] targetColumns;

		private KeyHolder() {
		}

		public KeyHolder(String keyId, String fromTable, String[] fromColumns, String targetTable, String[] targetColumns) {
		this.keyId = keyId;
		this.fromTable = fromTable;
		this.fromColumns = fromColumns;
		this.targetTable = targetTable;
		this.targetColumns = targetColumns;
		}

		public String getKeyId() {
		return keyId;
		}

		public String getFromTable() {
		return fromTable;
		}

		public String[] getFromColumns() {
		return fromColumns;
		}

		public String getTargetTable() {
		return targetTable;
		}

		public String[] getTargetColumns() {
		return targetColumns;
		}

		private String getColumnsString(String[] columns) {
		StringBuilder sb = new StringBuilder();
		boolean first = true;
		for (String column : columns) {
		if (!first) {
		sb.append(",");
		}
		sb.append(column);
		first = false;
		}
		return sb.toString();
		}

		@Override
		public String toString() {
		return String.format("[%s] %s(%s) -> %s(%s)", keyId,
		fromTable, getColumnsString(fromColumns),
		targetTable, getColumnsString(targetColumns));
		}
		}