Commit ae6327c1 authored by Sonia Zorba's avatar Sonia Zorba
Browse files

Added consistency check for missing keys

parent d58aff54
......@@ -10,6 +10,7 @@
* Fixed bug on consistency checking when a schema is missing
* Added warning in case of possible wrong source credentials selection (this is shown when consistency checking detect only the TAP_SCHEMA itself).
* Avoided connection timeout on the webapp when loading big TAP_SCHEMA schemas.
* Bugfix consistency checking: added check for missing keys.
## Version 1.0.4
......
......@@ -17,8 +17,7 @@ See also the [CHANGELOG]().
Edit configuration properties file (`src/main/resources/webapp.properties`):
ucd_service_url=<URL for the UCD REST web service>
credentials_config_path=<location where the web app will store the generated XML configuration>
password=<password for the TAP_SCHEMA Manager admin>
config_file_path=<location where the web app will store the generated XML configuration>
IA2 UCD service is at http://ia2-vo.oats.inaf.it:8080/ucd/
......
......@@ -46,18 +46,47 @@ import org.slf4j.LoggerFactory;
public class ConsistencyChecks implements Serializable {
private static final long serialVersionUID = 4412404312756740093L;
private final static Logger log = LoggerFactory.getLogger(ConsistencyChecks.class);
private final static Logger LOG = LoggerFactory.getLogger(ConsistencyChecks.class);
private static class UnexistingKeyColumn {
private final String keyId;
private final String fromColumn;
private final String targetColumn;
private UnexistingKeyColumn(String keyId, String fromColumn, String targetColumn) {
this.keyId = keyId;
this.fromColumn = fromColumn;
this.targetColumn = targetColumn;
}
public String getKeyId() {
return keyId;
}
public String getFromColumn() {
return fromColumn;
}
public String getTargetColumn() {
return targetColumn;
}
}
private final List<InconsistentValue> inconsistencies;
private final List<String> unexisingSchemas;
private final List<String> unexisingTables;
private final Map<String, String> unexisingColumns;
private final List<String> unexistingKeys;
private final List<UnexistingKeyColumn> unexistingKeyColumns;
public ConsistencyChecks() {
inconsistencies = new ArrayList<>();
unexisingSchemas = new ArrayList<>();
unexisingTables = new ArrayList<>();
unexisingColumns = new HashMap<>();
unexistingKeys = new ArrayList<>();
unexistingKeyColumns = new ArrayList<>();
}
public void addInconsistency(InconsistentValue problemDescription) {
......@@ -92,13 +121,28 @@ public class ConsistencyChecks implements Serializable {
unexisingColumns.put(completeTableName, columnName);
}
public void addUnexistingKey(String keyId) {
if (keyId == null) {
throw new IllegalArgumentException("key_id can't be null");
}
unexistingKeys.add(keyId);
}
public List<String> getUnexistingKeys() {
return unexistingKeys;
}
public void addUnexistingKeyColumn(String keyId, String fromColumn, String targetColumn) {
unexistingKeyColumns.add(new UnexistingKeyColumn(keyId, fromColumn, targetColumn));
}
private Set<String> getKeysToRemove(Connection conn, String tapSchemaNameEscaped, DatabaseType dbType, String like) throws SQLException {
Set<String> ret = new HashSet<>();
String query = String.format("SELECT key_id from %s.%s WHERE from_table LIKE ? OR target_table LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, like + "%");
ps.setString(2, like + "%");
log.debug("Executing query: {} [{}]", query, like);
LOG.debug("Executing query: {} [{}]", query, like);
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
ret.add(rs.getString("key_id"));
......@@ -143,7 +187,7 @@ public class ConsistencyChecks implements Serializable {
ps.setString(3, entry.getKey());
ps.setString(4, entry.getValue());
log.debug("Executing query {}", query);
LOG.debug("Executing query {}", query);
try (ResultSet rs = ps.executeQuery()) {
while (rs.next()) {
......@@ -153,8 +197,10 @@ public class ConsistencyChecks implements Serializable {
}
}
keysToRemoveIds.addAll(unexistingKeys);
conn.setAutoCommit(false);
log.debug("Starting transaction");
LOG.debug("Starting transaction");
try {
// Removing all key_columns
......@@ -162,7 +208,7 @@ public class ConsistencyChecks implements Serializable {
query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("key_columns", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, keyId);
log.debug("Executing query {} [{}]", query, keyId);
LOG.debug("Executing query {} [{}]", query, keyId);
ps.executeUpdate();
}
}
......@@ -172,7 +218,7 @@ public class ConsistencyChecks implements Serializable {
query = String.format("DELETE FROM %s.%s WHERE key_id = ?", tapSchemaNameEscaped, TSMUtil.escapeName("keys", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, keyId);
log.debug("Executing query {} [{}]", query, keyId);
LOG.debug("Executing query {} [{}]", query, keyId);
ps.executeUpdate();
}
}
......@@ -183,7 +229,7 @@ public class ConsistencyChecks implements Serializable {
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, entry.getKey());
ps.setString(2, entry.getValue());
log.debug("Executing query {} [{}, {}]", query, entry.getKey(), entry.getValue());
LOG.debug("Executing query {} [{}, {}]", query, entry.getKey(), entry.getValue());
ps.executeUpdate();
}
}
......@@ -191,7 +237,7 @@ public class ConsistencyChecks implements Serializable {
query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, table);
log.debug("Executing query {} [{}]", query, table);
LOG.debug("Executing query {} [{}]", query, table);
ps.executeUpdate();
}
}
......@@ -199,7 +245,7 @@ public class ConsistencyChecks implements Serializable {
query = String.format("DELETE FROM %s.%s WHERE table_name LIKE ?", tapSchemaNameEscaped, TSMUtil.escapeName("columns", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, schema + "%");
log.debug("Executing query {} [{}%]", query, schema);
LOG.debug("Executing query {} [{}%]", query, schema);
ps.executeUpdate();
}
}
......@@ -209,7 +255,7 @@ public class ConsistencyChecks implements Serializable {
query = String.format("DELETE FROM %s.%s WHERE table_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, table);
log.debug("Executing query {} [{}]", query, table);
LOG.debug("Executing query {} [{}]", query, table);
ps.executeUpdate();
}
}
......@@ -217,7 +263,7 @@ public class ConsistencyChecks implements Serializable {
query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("tables", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, schema);
log.debug("Executing query {} [{}]", query, schema);
LOG.debug("Executing query {} [{}]", query, schema);
ps.executeUpdate();
}
}
......@@ -227,19 +273,19 @@ public class ConsistencyChecks implements Serializable {
query = String.format("DELETE FROM %s.%s WHERE schema_name = ?", tapSchemaNameEscaped, TSMUtil.escapeName("schemas", dbType));
try (PreparedStatement ps = conn.prepareStatement(query)) {
ps.setString(1, schema);
log.debug("Executing query {} [{}]", query, schema);
LOG.debug("Executing query {} [{}]", query, schema);
ps.executeUpdate();
}
}
conn.commit();
} catch (SQLException e) {
log.error("Exception detected. Executing rollback!", e);
LOG.error("Exception detected. Executing rollback!", e);
try {
conn.rollback();
conn.setAutoCommit(true);
} catch (SQLException er) {
log.error("Exception during rollback", er);
LOG.error("Exception during rollback", er);
throw er;
}
}
......
......@@ -114,8 +114,8 @@ public class DaoKey {
String databaseName
= schemaName.equals(tapSchema.getName())
? dbWrapper.getTapSchemaCredentials().getDatabase()
: dbWrapper.getSourceCredentials().getDatabase();
? dbWrapper.getTapSchemaCredentials().getDatabase()
: dbWrapper.getSourceCredentials().getDatabase();
List<Key> schemaKeys = new ArrayList<>();
......@@ -221,6 +221,7 @@ public class DaoKey {
}
}
// Building query for the keys table
SelectQueryBuilder keysSelect = new SelectQueryBuilder(dbWrapper.getTapSchemaDatabaseType(), tapSchema, TapSchema.KEYS_TABLE) {
@Override
protected TapSchemaEntity getEntity(ResultSet rs) throws SQLException {
......@@ -228,6 +229,8 @@ public class DaoKey {
}
};
String queryKeys = keysSelect.getQuery();
// Building query for the key_columns table
SelectQueryBuilder keyColumnsSelect = new SelectQueryBuilder(dbWrapper.getTapSchemaDatabaseType(), tapSchema, TapSchema.KEY_COLUMNS_TABLE) {
@Override
protected TapSchemaEntity getEntity(ResultSet rs) throws SQLException {
......@@ -252,83 +255,84 @@ public class DaoKey {
while (rsKeys.next()) {
// Searching the keys.
String keyId = rsKeys.getString(Key.ID_KEY);
String fromTableCompleteNameSplit[] = rsKeys.getString(Key.FROM_TABLE_KEY).split(Pattern.quote("."));
String fromSchemaName = fromTableCompleteNameSplit[0];
String fromTableName = fromTableCompleteNameSplit[1];
Schema fromSchema = tapSchema.getChild(fromSchemaName);
if (fromSchema == null) {
throw new InconsistentTapSchemaException("Saved TAP_SCHEMA contains a key that is referred to a schema that wasn't added to that TAP_SCHEMA.");
}
Table fromTable = fromSchema.getChild(fromTableName);
if (fromTable == null) {
throw new InconsistentTapSchemaException("Saved TAP_SCHEMA contains a key that is referred to a table that wasn't added to that TAP_SCHEMA.");
}
String keyId = rsKeys.getString(Key.ID_KEY);
// ResultSet type and concurrency are necessary for PostgreSQL
try (PreparedStatement statementKeyColumns = conn.prepareStatement(queryKeyColumns, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)) {
statementKeyColumns.setString(1, keyId);
log.debug("Executing query {} [key_id={}]", queryKeyColumns, keyId);
try (ResultSet rsKeyColumns = statementKeyColumns.executeQuery()) {
for (Key fromKey : fromTable.getAllFromKeys()) {
boolean columnsFound = false;
for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
columnsFound = false;
rsKeyColumns.beforeFirst();
while (rsKeyColumns.next()) {
String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
if (keyColumn.getFromColumn().equals(fromColumn)
&& keyColumn.getTargetColumn().equals(targetColumn)) {
columnsFound = true;
break;
if (fromSchema == null) {
tapSchema.getConsistencyChecks().addUnexistingKey(keyId);
} else {
Table fromTable = fromSchema.getChild(fromTableName);
if (fromTable == null) {
tapSchema.getConsistencyChecks().addUnexistingKey(keyId);
} else {
// ResultSet type and concurrency are necessary for PostgreSQL
try (PreparedStatement statementKeyColumns = conn.prepareStatement(queryKeyColumns, ResultSet.TYPE_SCROLL_SENSITIVE, ResultSet.CONCUR_UPDATABLE)) {
statementKeyColumns.setString(1, keyId);
log.debug("Executing query {} [key_id={}]", queryKeyColumns, keyId);
try (ResultSet rsKeyColumns = statementKeyColumns.executeQuery()) {
for (Key fromKey : fromTable.getAllFromKeys()) {
boolean columnsFound = false;
for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
columnsFound = false;
rsKeyColumns.beforeFirst();
while (rsKeyColumns.next()) {
String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
if (keyColumn.getFromColumn().equals(fromColumn)
&& keyColumn.getTargetColumn().equals(targetColumn)) {
columnsFound = true;
break;
}
}
if (!columnsFound) {
break;
}
}
}
if (!columnsFound) {
break;
}
}
if (columnsFound) {
// all columns found --> key found!
if (columnsFound) {
// all columns found --> key found!
// Updating key
String keyDescription = rsKeys.getString(Key.DESCRIPTION_KEY);
String keyUtype = rsKeys.getString(Key.UTYPE_KEY);
// Updating key
String keyDescription = rsKeys.getString(Key.DESCRIPTION_KEY);
String keyUtype = rsKeys.getString(Key.UTYPE_KEY);
fromKey.initProperty(Key.ID_KEY, keyId);
fromKey.initProperty(Key.DESCRIPTION_KEY, keyDescription);
fromKey.initProperty(Key.UTYPE_KEY, keyUtype);
if (supportKeyID) {
fromKey.initProperty(Key.KEY_ID_KEY, TSMUtil.getObject(rsKeys, Key.KEY_ID_KEY, Long.class));
}
((KeyImpl) fromKey).setVisible(true);
// Updating key columns
for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
rsKeyColumns.beforeFirst();
while (rsKeyColumns.next()) {
String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
if (keyColumn.getFromColumn().equals(fromColumn)
&& keyColumn.getTargetColumn().equals(targetColumn)) {
keyColumn.initProperty(KeyColumn.KEY_ID_KEY, keyId);
if (supportKeyColumnID) {
keyColumn.initProperty(KeyColumn.KEY_COLUMN_ID_KEY, TSMUtil.getObject(rsKeyColumns, KeyColumn.KEY_COLUMN_ID_KEY, Long.class));
fromKey.initProperty(Key.ID_KEY, keyId);
fromKey.initProperty(Key.DESCRIPTION_KEY, keyDescription);
fromKey.initProperty(Key.UTYPE_KEY, keyUtype);
if (supportKeyID) {
fromKey.initProperty(Key.KEY_ID_KEY, TSMUtil.getObject(rsKeys, Key.KEY_ID_KEY, Long.class));
}
((KeyImpl) fromKey).setVisible(true);
// Updating key columns
for (KeyColumn keyColumn : fromKey.getKeyColumns()) {
rsKeyColumns.beforeFirst();
while (rsKeyColumns.next()) {
String fromColumn = rsKeyColumns.getString(KeyColumn.FROM_COLUMN_KEY);
String targetColumn = rsKeyColumns.getString(KeyColumn.TARGET_COLUMN_KEY);
if (keyColumn.getFromColumn().equals(fromColumn)
&& keyColumn.getTargetColumn().equals(targetColumn)) {
keyColumn.initProperty(KeyColumn.KEY_ID_KEY, keyId);
if (supportKeyColumnID) {
keyColumn.initProperty(KeyColumn.KEY_COLUMN_ID_KEY, TSMUtil.getObject(rsKeyColumns, KeyColumn.KEY_COLUMN_ID_KEY, Long.class));
}
break;
}
}
break;
}
break;
}
}
break;
}
}
}
......@@ -348,7 +352,7 @@ public class DaoKey {
break;
}
}
if (!keyIdFound) {
if (!keyIdFound && !tapSchema.getConsistencyChecks().getUnexistingKeys().contains(keyId)) {
String fromTableCompleteName = rsKeys.getString(Key.FROM_TABLE_KEY);
String targetTableCompleteName = rsKeys.getString(Key.TARGET_TABLE_KEY);
KeyImpl key = new KeyImpl(dbWrapper, tapSchema, fromTableCompleteName, targetTableCompleteName);
......
/*
* _____________________________________________________________________________
*
* INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
* Trieste INAF - IA2 Italian Center for Astronomical Archives
* _____________________________________________________________________________
*
* Copyright (C) 2016 Istituto Nazionale di Astrofisica
*
* This program is free software; you can redistribute it and/or modify it under
* the terms of the GNU General Public License Version 3 as published by the
* Free Software Foundation.
*
* This program is distributed in the hope that it will be useful, but WITHOUT
* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
* FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
* details.
*
* You should have received a copy of the GNU General Public License along with
* this program; if not, write to the Free Software Foundation, Inc., 51
* Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
*/
package it.inaf.ia2.tsm.api;
/**
*
* @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
*/
public class InconsistentTapSchemaException extends RuntimeException {
private static final long serialVersionUID = 2722256809774917529L;
public InconsistentTapSchemaException(String message) {
super(message);
}
}
......@@ -190,7 +190,11 @@ public class SchemaSelectionBean implements Serializable {
try {
loadedTapSchema = TapSchemaFactory.getTapSchema(TapSchemaVersion.TAP_SCHEMA_1_IA2, dbWrapper, selectedTAPSchema, true);
} catch (Throwable e) {
LOG.error("Exception caught", e);
loadingError = e.getMessage();
if(loadingError == null) {
loadingError = e.getClass().getCanonicalName();
}
}
loading = false;
}
......
......@@ -66,6 +66,15 @@
</ui:repeat>
</ul>
</h:panelGroup>
<h:panelGroup rendered="#{consistency.tapSchema.consistencyChecks.unexistingKeys.size() gt 0}">
<h2>Missing keys</h2>
<ul>
<ui:repeat value="#{consistency.tapSchema.consistencyChecks.unexistingKeys}" var="key">
<li>${key}</li>
</ui:repeat>
</ul>
</h:panelGroup>
<br/>
<h:panelGroup rendered="#{consistency.tapSchemaContainsOnlyTapSchema}" layout="block" class="alert alert-danger text-center">
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment