TapSchema.java

/* 
 * _____________________________________________________________________________
 * 
 * INAF - OATS National Institute for Astrophysics - Astronomical Observatory of
 * Trieste INAF - IA2 Italian Center for Astronomical Archives
 * _____________________________________________________________________________
 * 
 * Copyright (C) 2016 Istituto Nazionale di Astrofisica
 * 
 * This program is free software; you can redistribute it and/or modify it under
 * the terms of the GNU General Public License Version 3 as published by the
 * Free Software Foundation.
 * 
 * This program is distributed in the hope that it will be useful, but WITHOUT
 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
 * FOR A PARTICULAR PURPOSE. See the GNU General Public License for more
 * details.
 * 
 * You should have received a copy of the GNU General Public License along with
 * this program; if not, write to the Free Software Foundation, Inc., 51
 * Franklin Street, Fifth Floor, Boston, MA 02110-1301, USA.
 */
package it.inaf.ia2.tsm;

import it.inaf.ia2.tsm.datalayer.DBBroker;
import it.inaf.ia2.tsm.datalayer.DBBrokerFactory;
import it.inaf.ia2.tsm.datalayer.DBWrapper;
import it.inaf.ia2.tsm.datalayer.DataTypeMode;
import it.inaf.ia2.tsm.model.ColumnModel;
import it.inaf.ia2.tsm.model.TableModel;
import it.inaf.ia2.tsm.model.SchemaModel;
import it.inaf.ia2.tsm.model.SchemaModels;
import java.io.Serializable;
import java.sql.SQLException;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeMap;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;

/**
 * Contains both data models of the entities exposed by the TAP_SCHEMA and
 * method for managing them and edit the TAP_SCHEMA content.
 *
 * @author Sonia Zorba {@literal <zorba at oats.inaf.it>}
 */
public class TapSchema implements EntitiesContainer<Schema>, Serializable {

    public static final String STANDARD_TAP_SCHEMA_NAME = "TAP_SCHEMA";
    public static final String STANDARD_IVOA_SCHEMA_NAME = "ivoa";

    // Mandatory tables constants
    public static final String TABLES_TABLE = "tables";
    public static final String SCHEMAS_TABLE = "schemas";
    public static final String COLUMNS_TABLE = "columns";
    public static final String KEYS_TABLE = "keys";
    public static final String KEY_COLUMNS_TABLE = "key_columns";

    public static final String DESCRIPTION_KEY = "description";

    private static final long serialVersionUID = 1678083091602571256L;

    private static final Logger LOG = LoggerFactory.getLogger(TapSchema.class);

    private final Map<String, Schema> schemas;
    private final Set<Key> allKeys;

    private boolean loading;
    private DBWrapper dbWrapper;
    private String dbName;
    private String name;
    private String ivoaSchemaDBName;
    private String ivoaSchemaName;
    private boolean exists;
    private TapSchemaSettings settings;
    private DataTypeMode dataTypeMode;

    private transient DBBroker sourceDBBroker;
    private transient DBBroker tapSchemaDBBroker;

    private ConsistencyChecks consistencyChecks;

    /**
     * Returns the {@link DBBroker} for the database containing the astronomical
     * data and the ObsCore (called the <em>source</em> in TASMAN jargon).
     *
     * @see it.inaf.ia2.tsm.datalayer.DBWrapper
     */
    public final DBBroker getSourceDBBroker() {
        if (sourceDBBroker == null) {
            sourceDBBroker = DBBrokerFactory.getDBBroker(dbWrapper.getSourceDataSourceWrapper(), dataTypeMode);
        }
        return sourceDBBroker;
    }

    /**
     * Returns the {@link DBBroker} for the database containing the TAP_SCHEMA
     * schema.
     *
     * @see it.inaf.ia2.tsm.datalayer.DBWrapper
     */
    public final DBBroker getTapSchemaDBBroker() {
        if (tapSchemaDBBroker == null) {
            tapSchemaDBBroker = DBBrokerFactory.getDBBroker(dbWrapper.getTapSchemaDataSourceWrapper(), dataTypeMode);
        }
        return tapSchemaDBBroker;
    }

    /**
     * Only for serialization.
     */
    private TapSchema() {
        schemas = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
        allKeys = new HashSet<>();
    }

    /**
     * Default constructor.
     *
     * @param exists true if the TAP_SCHEMA has already been created, false if
     * must be created when the {@link #save() method is called}.
     */
    public TapSchema(DBWrapper dbWrapper, TapSchemaSettings settings, boolean exists) throws SQLException {
        this();

        this.dbWrapper = dbWrapper;
        this.exists = exists;
        this.settings = settings;
        // Don't change the instructions order!
        loadDBName();
        loadName();

        dataTypeMode = getTapSchemaModel().getDataTypeMode();

        load();
    }

    private void loadDBName() {
        // Detect if the TAP_SCHEMA version supports dbmodel
        SchemaModel tapSchemaModel = SchemaModels.getTapSchemaModel(settings.getTapSchemaVersion());
        boolean hasDBName = tapSchemaModel.getTable(SCHEMAS_TABLE).get("dbname") != null;
        if (hasDBName) {
            if (!STANDARD_TAP_SCHEMA_NAME.equals(settings.getTapSchemaName())) {
                dbName = settings.getTapSchemaName();
            }
            if (!STANDARD_IVOA_SCHEMA_NAME.equals(settings.getIvoaSchemaName())) {
                ivoaSchemaDBName = settings.getIvoaSchemaName();
            }
        }
    }

    private void loadName() {
        if (dbName != null) {
            name = STANDARD_TAP_SCHEMA_NAME;
        } else {
            name = settings.getTapSchemaName();
        }
        if (ivoaSchemaDBName != null) {
            ivoaSchemaName = STANDARD_IVOA_SCHEMA_NAME;
        } else {
            ivoaSchemaName = settings.getIvoaSchemaName();
        }
    }

    /**
     * Loads the TAP_SCHEMA information from the database. This method is called
     * in the constructor, but it can be called in every moment for reloading
     * database metadata (this is useful if some modifications have been
     * performed to the source schemata structure).
     */
    public final void load() throws SQLException {

        loading = true;
        // Initializing schemas map
        schemas.clear();
        for (String schemaName : getSourceDBBroker().getAllSchemaNames()) {
            schemas.put(schemaName, null);
        }
        schemas.put(getName(), null); // the TAP_SCHEMA contains itself
        if (settings.isHasObscore() && ivoaSchemaDBName != null) {
            schemas.put(getIvoaSchemaName(), null);
        }

        if (exists) {
            consistencyChecks = TapSchemaLoader.loadExistingTapSchema((this));
        }

        loading = false;

        checkKeys();
    }

    /**
     * Returns the {@link DBBroker} related to a given schema.
     */
    public DBBroker getDBBroker(String schemaName) {
        if (schemaName.equals(getName())) {
            return getTapSchemaDBBroker();
        } else {
            return getSourceDBBroker();
        }
    }

    /**
     * Returns the name of the TAP_SCHEMA schema, as exposed by itself.
     */
    public final String getName() {
        return name;
    }

    /**
     * Return the versions selected for this TAP_SCHEMA.
     */
    public String getVersion() {
        return settings.getTapSchemaVersion();
    }

    /**
     * Returns the {@link DataTypeMode} used by this TAP_SCHEMA.
     */
    public DataTypeMode getDataTypeMode() {
        return dataTypeMode;
    }

    private void loadSchemaKeysMetadata(String schemaName) throws SQLException {
        allKeys.addAll(getDBBroker(schemaName)
                .getKeys(this, schemaName, getRealSchemaName(schemaName)));
    }

    /**
     * Returns all {@link Key} entities loaded by this instance; this include
     * both visible and hidden keys (that are keys that must be exposed by the
     * TAP_SCHEMA and keys that mustn't).
     */
    public Set<Key> getAllKeys() {
        return allKeys;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public final Schema addChild(String schemaName) throws SQLException {
        LOG.debug("Adding schema {}", schemaName);

        Schema schema;

        if (!schemas.containsKey(schemaName)) {
            schema = null;
        } else {

            schema = schemas.get(schemaName);

            if (schema == null) {
                schema = new Schema(this, schemaName);
                schema.setStatus(Status.ADDED_NOT_PERSISTED);
                schemas.put(schemaName, schema);
                loadSchemaKeysMetadata(schemaName);
            } else {
                switch (schema.getStatus()) {
                    case TO_REMOVE:
                        schema.setStatus(Status.ADDED_PERSISTED);
                        break;
                    case REMOVED_NOT_PERSISTED:
                        schema.setStatus(Status.ADDED_NOT_PERSISTED);
                        break;
                    default:
                        throw new IllegalArgumentException("Cannot add the schema " + schemaName + ". Invalid status. Schema status is " + schema.getStatus());
                }
            }

            checkKeys();
        }

        return schema;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public void removeChild(String schemaName) {
        LOG.debug("Removing schema {}", schemaName);

        if (!schemas.containsKey(schemaName)) {
            throw new IllegalArgumentException("The database doesn't contains a schema named " + schemaName);
        }

        Schema schema = schemas.get(schemaName);
        if (schema == null || schema.getStatus() == Status.LOADED) {
            throw new IllegalArgumentException("Cannot remove the schema " + schemaName + ". It has never been added.");
        }

        if (schema.getStatus() == Status.ADDED_NOT_PERSISTED) {
            schema.setStatus(Status.REMOVED_NOT_PERSISTED);
        } else if (schema.getStatus() == Status.ADDED_PERSISTED) {
            schema.setStatus(Status.TO_REMOVE);
        }

        checkKeys();
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public final Schema getChild(String childName, Status... statuses) {
        return TSMUtil.getChild(schemas, childName, statuses);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<Schema> getChildren(Status... statuses) {
        return TSMUtil.getChildrenByStatus(schemas.values(), statuses);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<String> getAddableChildrenNames() {
        List<String> addables = TSMUtil.getAddableChildrenNames(schemas);
        addables = new ArrayList<>(addables); // list returned by previous method is unmodified
        addables.remove(getName()); // TAP_SCHEMA insertion must be managed by TASMAN
        return addables;
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public boolean isAddable(String childName) {
        return schemas.containsKey(childName);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<Schema> getAddedChildren() {
        return getChildren(Status.ADDED_PERSISTED, Status.ADDED_NOT_PERSISTED);
    }

    /**
     * {@inheritDoc}
     */
    @Override
    public List<Schema> getAddedOrRemovedChildren() {
        return getChildren(Status.ADDED_PERSISTED, Status.ADDED_NOT_PERSISTED, Status.TO_REMOVE, Status.REMOVED_NOT_PERSISTED);
    }

    /**
     * This method has to be used after TAP_SCHEMA modifications are committed
     * to the database, in order to remove from the memory the schemas with
     * status {@code Status.TO_REMOVE} or {@code Status.REMOVED_NOT_PERSISTED}.
     */
    public void cleanSchema(String schemaName) {
        if (!schemas.containsKey(schemaName)) {
            throw new IllegalArgumentException("The TAP_SCHEMA doesn't contain the schema " + schemaName);
        }
        schemas.put(schemaName, null);
    }

    /**
     * Returns the {@link SchemaModel} for the {@code ivoa} schema if the
     * {@code ObsCore} table must be managed by this {@code TapSchema}, returns
     * null otherwise.
     */
    public SchemaModel getIvoaSchemaModel() {
        if (settings.isHasObscore()) {
            return SchemaModels.getIvoaSchemaModel(settings.getObscoreVersion());
        }
        return null;
    }

    /**
     * Add an entire schema to the TAP_SCHEMA, including all all its tables and
     * columns children.
     */
    public void addEntireSchema(String schemaName) throws SQLException {
        Schema schema = addChild(schemaName);
        for (String tableName : schema.getAddableChildrenNames()) {
            Table table = schema.addChild(tableName);
            for (String columnName : table.getAddableChildrenNames()) {
                table.addChild(columnName);
            }
        }
    }

    /**
     * Save or update the TAP_SCHEMA changes into the database.
     */
    public void save() throws SQLException {

        DBBroker broker = getTapSchemaDBBroker();

        if (!exists) {
            SchemaModel tapSchemaModel = getTapSchemaModel();
            broker.createTapSchemaStructure(getRealName(), tapSchemaModel);

            // Adding TAP_SCHEMA into TAP_SCHEMA
            addEntireSchema(getName());
            fillColumnProperties(tapSchemaModel, getName());

            if (settings.isHasObscore()) {
                createAndAddIvoaSchema();
            }
        }

        fillKeyIds();

        broker.save(this);
        exists = true;

        // Clean inconsistency
        consistencyChecks = null;
    }

    public void createAndAddIvoaSchema() throws SQLException {
        SchemaModel ivoaSchemaModel = getIvoaSchemaModel();

        // ivoa schema has to be created into source database
        getSourceDBBroker().createIvoaSchemaStructure(ivoaSchemaModel, getRealSchemaName(ivoaSchemaModel.getName()));
        // Initializing ivoa schema slot in schemata maps
        schemas.put(ivoaSchemaModel.getName(), null);

        // Add ivoa schema into TAP_SCHEMA
        addEntireSchema(ivoaSchemaModel.getName());
        fillColumnsProperties(ivoaSchemaModel);
    }

    /**
     * Retrieve the maximum key id from all the keys that are added into the
     * TAP_SCHEMA.
     *
     * @return the maximum key id, if it exists, zero otherwise.
     */
    private int getMaxKeyId() {
        int maxKeyId = 0;
        for (Key key : allKeys) {
            if (key.getId() != null) {
                int keyId = Integer.parseInt(key.getId());
                if (keyId > maxKeyId) {
                    maxKeyId = keyId;
                }
            }
        }
        return maxKeyId;
    }

    private void fillKeyIds() {

        List<Key> newKeys = new ArrayList<>();
        for (Key key : allKeys) {
            if (key.isVisible() && key.getId() == null) {
                newKeys.add(key);
            }
        }

        int maxKeyId = getMaxKeyId();
        for (Key newKey : newKeys) {
            maxKeyId++;
            newKey.setId(String.valueOf(maxKeyId));
        }
    }

    /**
     * Set keys visibility based on other entities visibility (a key is visible
     * if all schemas, tables and columns involved have
     * {@link Status} {@code ADDED_PERSISTED} or {@code ADDED_NOT_PERSISTED}).
     */
    public final void checkKeys() {
        if (!loading) {

            for (Key key : allKeys) {

                // Check if key should be exposed in TAP_SCHEMA
                boolean keyVisible = true;

                for (KeyColumn keyColumn : key.getKeyColumns()) {
                    String schemaName = keyColumn.getParent().getFromSchemaName();
                    String tableName = keyColumn.getParent().getFromTableSimpleName();
                    String columnName = keyColumn.getFromColumn();
                    if (!isColumnVisible(schemaName, tableName, columnName)) {
                        keyVisible = false;
                        break;
                    }

                    schemaName = keyColumn.getParent().getTargetSchemaName();
                    tableName = keyColumn.getParent().getTargetTableSimpleName();
                    columnName = keyColumn.getTargetColumn();
                    if (!isColumnVisible(schemaName, tableName, columnName)) {
                        keyVisible = false;
                        break;
                    }
                }

                // TODO: use status instead of set visibile [?]
                key.setVisible(keyVisible);
            }
        }
    }

    /**
     * Print all TAP_SCHEMA tree (useful for debugging).
     */
    @Override
    public String toString() {

        StringBuilder sb = new StringBuilder("\n");

        sb.append(String.format(">> TAP_SCHEMA %s <<\n", getName()));

        for (Schema schema : getChildren()) {
            sb.append("--");
            sb.append(schema.getName());
            sb.append(String.format(" [%s]", schema.getStatus()));
            sb.append("\n");
            List<Table> tables = schema.getChildren();
            for (int i = 0; i < tables.size(); i++) {
                Table table = tables.get(i);
                sb.append("   |--");
                sb.append(table.getName());
                sb.append(String.format(" [%s]", table.getStatus()));
                sb.append("\n");

                String padder = i < tables.size() - 1 ? "   |   " : "       ";

                for (Column column : table.getChildren()) {
                    sb.append(padder);
                    sb.append("|--");
                    sb.append(column.getName());
                    sb.append(String.format(" [%s]", column.getStatus()));
                    sb.append("\n");
                }

                sb.append("\n");
            }
        }

        sb.append("** Keys **\n");
        for (Key key : getVisibileKeys()) {
            sb.append(key);
            sb.append("\n");
        }

        return sb.toString();
    }

    /**
     * Tells if a schema has to be exposed by the TAP_SCHEMA.
     */
    public boolean isSchemaVisible(String schemaName) {
        Schema schema = schemas.get(schemaName);
        if (schema == null) {
            return false;
        }
        return schema.getStatus() == Status.ADDED_PERSISTED
                || schema.getStatus() == Status.ADDED_NOT_PERSISTED;
    }

    /**
     * Tells if a table has to be exposed by the TAP_SCHEMA.
     */
    public boolean isTableVisible(String schemaName, String tableName) {
        if (!isSchemaVisible(schemaName)) {
            return false;
        }

        Table table = schemas.get(schemaName).getChild(tableName);
        if (table == null) {
            return false;
        }

        if (table.getStatus() == Status.ADDED_PERSISTED
                || table.getStatus() == Status.ADDED_NOT_PERSISTED) {
            return isSchemaVisible(schemaName);
        }

        return false;
    }

    /**
     * Tells if a column has to be exposed by the TAP_SCHEMA.
     */
    public boolean isColumnVisible(String schemaName, String tableName, String columnName) {
        if (!isTableVisible(schemaName, tableName)) {
            return false;
        }

        Column column = schemas.get(schemaName).getChild(tableName).getChild(columnName);
        if (column == null) {
            return false;
        }

        if (column.getStatus() == Status.ADDED_PERSISTED
                || column.getStatus() == Status.ADDED_NOT_PERSISTED) {
            return isTableVisible(schemaName, tableName);
        }

        return false;
    }

    /**
     * Tells if the TAP_SCHEMA schema was already written into the database or
     * it has to be created when the {@link #save()} method will be called.
     */
    public boolean exists() {
        return exists;
    }

    /**
     * Returns the result of the consistency checking performed during the
     * TAP_SCHEMA loading.
     */
    public ConsistencyChecks getConsistencyChecks() {
        return consistencyChecks;
    }

    /**
     * Returns the {@link SchemaModel} for the TAP_SCHEMA schema.
     */
    public final SchemaModel getTapSchemaModel() {
        SchemaModel tapSchemaModel = SchemaModels.getTapSchemaModel(getVersion());
        if (tapSchemaModel == null) {
            throw new IllegalStateException("TAP_SCHEMA model is null for version " + getVersion());
        }
        return tapSchemaModel;
    }

    /**
     * Returns the {@link TableModel} for a TAP_SCHEMA table.
     */
    public final TableModel getTableModel(String tableName) {
        return getTapSchemaModel().getTable(tableName);
    }

    /**
     * Returns the metadata of a schema managed by this TAP_SCHEMA.
     */
    public Map<String, Object> getSchemaMetadata(String schemaName) {
        Map<String, Object> metadata = new HashMap<>();
        metadata.put(Schema.SCHEMA_NAME_KEY, schemaName);
        String dbNameMetadata = null;
        if (dbName != null && schemaName.equals(STANDARD_TAP_SCHEMA_NAME)) {
            dbNameMetadata = dbName;
        }
        metadata.put(Schema.DBNAME, dbNameMetadata);
        return metadata;
    }

    /**
     * Returns all the keys that are currently exposed by this TAP_SCHEMA.
     */
    public List<Key> getVisibileKeys() {
        List<Key> visibleKeys = new ArrayList<>();
        for (Key key : allKeys) {
            if (key.isVisible()) {
                visibleKeys.add(key);
            }
        }
        return visibleKeys;
    }

    private Integer getIntAsBool(Boolean value) {
        if (value == null) {
            return null;
        }
        return value ? 1 : 0;
    }

    /**
     * Tells if the {@code ObsCore} table should be managed or not.
     */
    public boolean isHasObscore() {
        return settings.isHasObscore();
    }

    private void fillColumnProperties(SchemaModel schemaModel, String schemaName) {

        // check only on std, but valid also for principal (it depends on TS version)
        boolean useIntegerAsBool = getTapSchemaModel().getTable(COLUMNS_TABLE).get(Column.STD_KEY).getJavaType() == Integer.class;

        Schema schema = getChild(schemaName);
        schema.setValue(DESCRIPTION_KEY, schemaModel.getDescription());
        for (TableModel tableModel : schemaModel.getTables()) {
            Table table = schema.getChild(tableModel.getName());
            table.setValue(DESCRIPTION_KEY, tableModel.getDescription());
            for (ColumnModel columnModel : tableModel.getColumns()) {
                Column column = table.getChild(columnModel.getName());
                if (!columnModel.isMandatory() && column == null) {
                    // column could be null if it is not mandatory
                    continue;
                }
                column.setValue(DESCRIPTION_KEY, columnModel.getDescription());
                column.setValue(Column.UCD_KEY, columnModel.getUcd());
                column.setValue(Column.UNIT_KEY, columnModel.getUnit());
                column.setValue(Column.UTYPE_KEY, columnModel.getUtype());

                Object compatibleStd = useIntegerAsBool ? getIntAsBool(columnModel.isStandard()) : columnModel.isStandard();
                Object compatiblePrincipal = useIntegerAsBool ? getIntAsBool(columnModel.isPrincipal()) : columnModel.isPrincipal();
                column.setValue(Column.STD_KEY, compatibleStd);
                column.setValue(Column.PRINCIPAL_KEY, compatiblePrincipal);
            }
        }
    }

    /**
     * Fills descriptions of the TAP_SCHEMA schema entities for a given
     * SchemaModel (TAP_SCHEMA or ivoa).
     */
    public void fillColumnsProperties(SchemaModel schemaModel) {
        fillColumnProperties(schemaModel, schemaModel.getName());
    }

    /**
     * Returns the TAP_SCHEMA schema {@code dbname} property, used to allow the
     * schema renaming supported by taplib (in this way the TAP_SCHEMA could be
     * exposed using the standard name, even if it is stored with a different
     * named into the database). This value is null if the TAP_SCHEMA version
     * doesn't support the {@code dbname} column or if the schema name is
     * already the standard value.
     */
    public String getDBName() {
        return dbName;
    }

    /**
     * Returns the ivoa schema {@code dbname} property, used to allow the schema
     * renaming supported by taplib.
     *
     * @see #getDBName()
     */
    public String getIvoaSchemaDBName() {
        return ivoaSchemaDBName;
    }

    /**
     * Returns the name of the {@code ivoa} schema, as exposed by the
     * TAP_SCHEMA.
     */
    public String getIvoaSchemaName() {
        return ivoaSchemaName;
    }

    /**
     * Returns the real name of the TAP_SCHEMA schema, as seen by the database
     * (useful when schema renaming has been configured).
     */
    public String getRealName() {
        return getRealSchemaName(getName());
    }

    /**
     * Returns the real name of a schema exposed by the TAP_SCHEMA, as seen by
     * the database (useful when schema renaming has been configured).
     */
    public String getRealSchemaName(String schemaName) {
        if (dbName != null && STANDARD_TAP_SCHEMA_NAME.equals(schemaName)) {
            return dbName;
        }
        if (ivoaSchemaDBName != null && STANDARD_IVOA_SCHEMA_NAME.equals(schemaName)) {
            return ivoaSchemaDBName;
        }
        return schemaName;
    }
}