Added consistency check for missing keys (ae6327c1) · Commits · Marco Molinaro / tap_schema_manager

CHANGELOG.md

+1 −0

Original line number	Diff line number	Diff line
		@@ -10,6 +10,7 @@
		* Fixed bug on consistency checking when a schema is missing
		* Added warning in case of possible wrong source credentials selection (this is shown when consistency checking detect only the TAP_SCHEMA itself).
		* Avoided connection timeout on the webapp when loading big TAP_SCHEMA schemas.
		* Bugfix consistency checking: added check for missing keys.

		## Version 1.0.4

README.md

+1 −2

Original line number	Diff line number	Diff line
		@@ -17,8 +17,7 @@ See also the [CHANGELOG]().
		Edit configuration properties file (`src/main/resources/webapp.properties`):

		ucd_service_url=<URL for the UCD REST web service>
		credentials_config_path=<location where the web app will store the generated XML configuration>
		password=<password for the TAP_SCHEMA Manager admin>
		config_file_path=<location where the web app will store the generated XML configuration>

		IA2 UCD service is at http://ia2-vo.oats.inaf.it:8080/ucd/

TASMAN-core/src/main/java/it/inaf/ia2/tsm/api/ConsistencyChecks.java

+60 −14

Original line number	Diff line number	Diff line
		@@ -46,18 +46,47 @@ import org.slf4j.LoggerFactory;
		public class ConsistencyChecks implements Serializable {

		private static final long serialVersionUID = 4412404312756740093L;
		private final static Logger log = LoggerFactory.getLogger(ConsistencyChecks.class);
		private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class);

		private static class UnexistingKeyColumn {

		private final String keyId;
		private final String fromColumn;
		private final String targetColumn;

		private UnexistingKeyColumn(String keyId, String fromColumn, String targetColumn) {
		this.keyId = keyId;
		this.fromColumn = fromColumn;
		this.targetColumn = targetColumn;
		}

		public String getKeyId() {
		return keyId;
		}

		public String getFromColumn() {
		return fromColumn;
		}

		public String getTargetColumn() {
		return targetColumn;
		}
		}

		private final List<InconsistentValue> inconsistencies;
		private final List<String> unexisingSchemas;
		private final List<String> unexisingTables;
		private final Map<String, String> unexisingColumns;
		private final List<String> unexistingKeys;
		private final List<UnexistingKeyColumn> unexistingKeyColumns;

		public ConsistencyChecks() {
		inconsistencies = new ArrayList<>();
		unexisingSchemas = new ArrayList<>();
		unexisingTables = new ArrayList<>();
		unexisingColumns = new HashMap<>();
		unexistingKeys = new ArrayList<>();
		unexistingKeyColumns = new ArrayList<>();
		}

		public void addInconsistency(InconsistentValue problemDescription) {
		@@ -92,13 +121,28 @@ public class ConsistencyChecks implements Serializable {
		unexisingColumns.put(completeTableName, columnName);
		}

		public void addUnexistingKey(String keyId) {
		if (keyId == null) {
		throw new IllegalArgumentException("key_id can't be null");
		}
		unexistingKeys.add(keyId);
		}

		public List<String> getUnexistingKeys() {
		return unexistingKeys;
		}

		public void addUnexistingKeyColumn(String keyId, String fromColumn, String targetColumn) {
		unexistingKeyColumns.add(new UnexistingKeyColumn(keyId, fromColumn, targetColumn));
		}

		private Set<String> getKeysToRemove(Connection conn, String tapSchemaNameEscaped, DatabaseType dbType, String like) throws SQLException {
		Set<String> ret = new HashSet<>();
		String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, like + "%");
		ps.setString(2, like + "%");
		log.debug("Executing query: {} [{}]", query, like);
		LOG.debug("Executing query: {} [{}]", query, like);
		try (ResultSet rs = ps.executeQuery()) {
		while (rs.next()) {
		ret.add(rs.getString("key_id"));
		@@ -143,7 +187,7 @@ public class ConsistencyChecks implements Serializable {
		ps.setString(3, entry.getKey());
		ps.setString(4, entry.getValue());

		log.debug("Executing query {}", query);
		LOG.debug("Executing query {}", query);

		try (ResultSet rs = ps.executeQuery()) {
		while (rs.next()) {
		@@ -153,8 +197,10 @@ public class ConsistencyChecks implements Serializable {
		}
		}

		keysToRemoveIds.addAll(unexistingKeys);

		conn.setAutoCommit(false);
		log.debug("Starting transaction");
		LOG.debug("Starting transaction");

		try {
		// Removing all key_columns
		@@ -162,7 +208,7 @@ public class ConsistencyChecks implements Serializable {
		query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("key_columns", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, keyId);
		log.debug("Executing query {} [{}]", query, keyId);
		LOG.debug("Executing query {} [{}]", query, keyId);
		ps.executeUpdate();
		}
		}
		@@ -172,7 +218,7 @@ public class ConsistencyChecks implements Serializable {
		query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, keyId);
		log.debug("Executing query {} [{}]", query, keyId);
		LOG.debug("Executing query {} [{}]", query, keyId);
		ps.executeUpdate();
		}
		}
		@@ -183,7 +229,7 @@ public class ConsistencyChecks implements Serializable {
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, entry.getKey());
		ps.setString(2, entry.getValue());
		log.debug("Executing query {} [{}, {}]", query, entry.getKey(), entry.getValue());
		LOG.debug("Executing query {} [{}, {}]", query, entry.getKey(), entry.getValue());
		ps.executeUpdate();
		}
		}
		@@ -191,7 +237,7 @@ public class ConsistencyChecks implements Serializable {
		query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, table);
		log.debug("Executing query {} [{}]", query, table);
		LOG.debug("Executing query {} [{}]", query, table);
		ps.executeUpdate();
		}
		}
		@@ -199,7 +245,7 @@ public class ConsistencyChecks implements Serializable {
		query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, schema + "%");
		log.debug("Executing query {} [{}%]", query, schema);
		LOG.debug("Executing query {} [{}%]", query, schema);
		ps.executeUpdate();
		}
		}
		@@ -209,7 +255,7 @@ public class ConsistencyChecks implements Serializable {
		query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, table);
		log.debug("Executing query {} [{}]", query, table);
		LOG.debug("Executing query {} [{}]", query, table);
		ps.executeUpdate();
		}
		}
		@@ -217,7 +263,7 @@ public class ConsistencyChecks implements Serializable {
		query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, schema);
		log.debug("Executing query {} [{}]", query, schema);
		LOG.debug("Executing query {} [{}]", query, schema);
		ps.executeUpdate();
		}
		}
		@@ -227,19 +273,19 @@ public class ConsistencyChecks implements Serializable {
		query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("schemas", dbType));
		try (PreparedStatement ps = conn.prepareStatement(query)) {
		ps.setString(1, schema);
		log.debug("Executing query {} [{}]", query, schema);
		LOG.debug("Executing query {} [{}]", query, schema);
		ps.executeUpdate();
		}
		}

		conn.commit();
		} catch (SQLException e) {
		log.error("Exception detected. Executing rollback!", e);
		LOG.error("Exception detected. Executing rollback!", e);
		try {
		conn.rollback();
		conn.setAutoCommit(true);
		} catch (SQLException er) {
		log.error("Exception during rollback", er);
		LOG.error("Exception during rollback", er);
		throw er;
		}
		}

TASMAN-core/src/main/java/it/inaf/ia2/tsm/api/DaoKey.java

+70 −66

Original line number	Diff line number	Diff line
		@@ -114,8 +114,8 @@ public class DaoKey {

		String databaseName
		= schemaName.equals(tapSchema.getName())
		? dbWrapper.getTapSchemaCredentials().getDatabase()
		: dbWrapper.getSourceCredentials().getDatabase();
		? dbWrapper.getTapSchemaCredentials().getDatabase()
		: dbWrapper.getSourceCredentials().getDatabase();

		List<Key> schemaKeys = new ArrayList<>();

		@@ -221,6 +221,7 @@ public class DaoKey {
		}
		}

		// Building query for the keys table
		SelectQueryBuilder keysSelect = new SelectQueryBuilder(dbWrapper.getTapSchemaDatabaseType(), tapSchema, TapSchema.KEYS_TABLE) {
		@Override
		protected TapSchemaEntity getEntity(ResultSet rs) throws SQLException {
		@@ -228,6 +229,8 @@ public class DaoKey {
		}
		};
		String queryKeys = keysSelect.getQuery();

		// Building query for the key_columns table
		SelectQueryBuilder keyColumnsSelect = new SelectQueryBuilder(dbWrapper.getTapSchemaDatabaseType(), tapSchema, TapSchema.KEY_COLUMNS_TABLE) {
		@Override
		protected TapSchemaEntity getEntity(ResultSet rs) throws SQLException {
		@@ -252,83 +255,84 @@ public class DaoKey {
		while (rsKeys.next()) {
		// Searching the keys.

		String keyId = rsKeys.getString(Key.ID_KEY);
		String fromTableCompleteNameSplit[] = rsKeys.getString(Key.FROM_TABLE_KEY).split(Pattern.quote("."));
		String fromSchemaName = fromTableCompleteNameSplit[0];
		String fromTableName = fromTableCompleteNameSplit[1];

		Schema fromSchema = tapSchema.getChild(fromSchemaName);
		if (fromSchema == null) {
		throw new InconsistentTapSchemaException("Saved TAP_SCHEMA contains a key that is referred to a schema that wasn't added to that TAP_SCHEMA.");
		}

		Table fromTable = fromSchema.getChild(fromTableName);
		if (fromTable == null) {
		throw new InconsistentTapSchemaException("Saved TAP_SCHEMA contains a key that is referred to a table that wasn't added to that TAP_SCHEMA.");
		}

		String keyId = rsKeys.getString(Key.ID_KEY);
		// ResultSet type and concurrency are necessary for PostgreSQL
		try (PreparedStatement statementKeyColumns = conn.prepareStatement(queryKeyColumns, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)) {

		statementKeyColumns.setString(1, keyId);
		log.debug("Executing query {} [key_id={}]", queryKeyColumns, keyId);

		try (ResultSet rsKeyColumns = statementKeyColumns.executeQuery()) {
		for (Key fromKey : fromTable.getAllFromKeys()) {

		boolean columnsFound = false;

		for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
		columnsFound = false;

		rsKeyColumns.beforeFirst();
		while (rsKeyColumns.next()) {
		String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
		String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
		if (keyColumn.getFromColumn().equals(fromColumn)
		&& keyColumn.getTargetColumn().equals(targetColumn)) {
		columnsFound = true;
		break;
		if (fromSchema == null) {
		tapSchema.getConsistencyChecks().addUnexistingKey(keyId);
		} else {
		Table fromTable = fromSchema.getChild(fromTableName);
		if (fromTable == null) {
		tapSchema.getConsistencyChecks().addUnexistingKey(keyId);
		} else {
		// ResultSet type and concurrency are necessary for PostgreSQL
		try (PreparedStatement statementKeyColumns = conn.prepareStatement(queryKeyColumns, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)) {

		statementKeyColumns.setString(1, keyId);
		log.debug("Executing query {} [key_id={}]", queryKeyColumns, keyId);

		try (ResultSet rsKeyColumns = statementKeyColumns.executeQuery()) {
		for (Key fromKey : fromTable.getAllFromKeys()) {

		boolean columnsFound = false;

		for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
		columnsFound = false;

		rsKeyColumns.beforeFirst();
		while (rsKeyColumns.next()) {
		String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
		String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
		if (keyColumn.getFromColumn().equals(fromColumn)
		&& keyColumn.getTargetColumn().equals(targetColumn)) {
		columnsFound = true;
		break;
		}
		}
		if (!columnsFound) {
		break;
		}
		}
		}
		if (!columnsFound) {
		break;
		}
		}

		if (columnsFound) {
		// all columns found --> key found!
		if (columnsFound) {
		// all columns found --> key found!

		// Updating key
		String keyDescription = rsKeys.getString(Key.DESCRIPTION_KEY);
		String keyUtype = rsKeys.getString(Key.UTYPE_KEY);
		// Updating key
		String keyDescription = rsKeys.getString(Key.DESCRIPTION_KEY);
		String keyUtype = rsKeys.getString(Key.UTYPE_KEY);

		fromKey.initProperty(Key.ID_KEY, keyId);
		fromKey.initProperty(Key.DESCRIPTION_KEY, keyDescription);
		fromKey.initProperty(Key.UTYPE_KEY, keyUtype);
		if (supportKeyID) {
		fromKey.initProperty(Key.KEY_ID_KEY, TSMUtil.getObject(rsKeys, Key.KEY_ID_KEY, Long.class));
		}
		((KeyImpl) fromKey).setVisible(true);

		// Updating key columns
		for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
		rsKeyColumns.beforeFirst();
		while (rsKeyColumns.next()) {
		String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
		String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
		if (keyColumn.getFromColumn().equals(fromColumn)
		&& keyColumn.getTargetColumn().equals(targetColumn)) {
		keyColumn.initProperty(KeyColumn.KEY_ID_KEY, keyId);
		if (supportKeyColumnID) {
		keyColumn.initProperty(KeyColumn.KEY_COLUMN_ID_KEY, TSMUtil.getObject(rsKeyColumns, KeyColumn.KEY_COLUMN_ID_KEY, Long.class));
		fromKey.initProperty(Key.ID_KEY, keyId);
		fromKey.initProperty(Key.DESCRIPTION_KEY, keyDescription);
		fromKey.initProperty(Key.UTYPE_KEY, keyUtype);
		if (supportKeyID) {
		fromKey.initProperty(Key.KEY_ID_KEY, TSMUtil.getObject(rsKeys, Key.KEY_ID_KEY, Long.class));
		}
		((KeyImpl) fromKey).setVisible(true);

		// Updating key columns
		for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
		rsKeyColumns.beforeFirst();
		while (rsKeyColumns.next()) {
		String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
		String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
		if (keyColumn.getFromColumn().equals(fromColumn)
		&& keyColumn.getTargetColumn().equals(targetColumn)) {
		keyColumn.initProperty(KeyColumn.KEY_ID_KEY, keyId);
		if (supportKeyColumnID) {
		keyColumn.initProperty(KeyColumn.KEY_COLUMN_ID_KEY, TSMUtil.getObject(rsKeyColumns, KeyColumn.KEY_COLUMN_ID_KEY, Long.class));
		}
		break;
		}
		}
		break;
		}

		break;
		}
		}

		break;
		}
		}
		}
		@@ -348,7 +352,7 @@ public class DaoKey {
		break;
		}
		}
		if (!keyIdFound) {
		if (!keyIdFound && !tapSchema.getConsistencyChecks().getUnexistingKeys().contains(keyId)) {
		String fromTableCompleteName = rsKeys.getString(Key.FROM_TABLE_KEY);
		String targetTableCompleteName = rsKeys.getString(Key.TARGET_TABLE_KEY);
		KeyImpl key = new KeyImpl(dbWrapper, tapSchema, fromTableCompleteName, targetTableCompleteName);

TASMAN-core/src/main/java/it/inaf/ia2/tsm/api/InconsistentTapSchemaException.java

deleted100644 → 0

+0 −36

Original line number	Diff line number	Diff line
		/*
		* _____________________________________________________________________________
		*
		* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
		* Trieste INAF - IA2 Italian Center for Astronomical Archives
		* _____________________________________________________________________________
		*
		* Copyright (C) 2016 Istituto Nazionale di Astrofisica
		*
		* This program is free software; you can redistribute it and/or modify it under
		* the terms of the GNU General Public License Version 3 as published by the
		* Free Software Foundation.
		*
		* This program is distributed in the hope that it will be useful, but WITHOUT
		* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
		* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
		* details.
		*
		* You should have received a copy of the GNU General Public License along with
		* this program; if not, write to the Free Software Foundation, Inc., 51
		* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
		*/
		package it.inaf.ia2.tsm.api;

		/**
		*
		* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
		*/
		public class InconsistentTapSchemaException extends RuntimeException {

		private static final long serialVersionUID = 2722256809774917529L;

		public InconsistentTapSchemaException(String message) {
		super(message);
		}
		}