implements dataunitsum calculation and verification (--dataunitsum switch) (3240fb04) · Commits · IA2 / fits_importer

Makefile

+3 −1

Original line number	Diff line number	Diff line
		@@ -9,6 +9,7 @@ TAR_NAME := `basename $(PWD)`
		INC_DIR=/usr/local/omniORB/include \
		/usr/local/zeromq/include/zmq \
		/usr/local/tango/include/tango \
		/usr/local/include/tango \
		/usr/local/boost/include \
		/usr/local/soci/include \
		/usr/local/soci/include/soci \
		@@ -21,6 +22,7 @@ INC_DIR=/usr/local/omniORB/include \
		LIB_DIR=/usr/local/omniORB/lib \
		/usr/local/zeromq/lib \
		/usr/local/tango/lib \
		/usr/local/lib \
		/usr/local/boost/lib \
		/usr/local/soci/lib64 \
		/usr/local/CCfits/lib \
		@@ -31,7 +33,7 @@ CXX_DEBUG_FLAGS=-g -DVERBOSE_DEBUG
		CXX_RELEASE_FLAGS=-O3
		CXX_DEFAULT_FLAGS=-c -Wall -Wextra -std=c++0x -DHAVE_BOOST
		LDFLAGS=-Wall -lomniORB4 -lomniDynamic4 -lCOS4 -lomnithread -ltango -llog4tango \
		-lsoci_core -lsoci_mysql -lCCfits -lboost_thread -lboost_filesystem \
		-lsoci_core -lsoci_mysql -lcfitsio -lCCfits -lboost_thread -lboost_filesystem \
		-lboost_system -lboost_iostreams -lboost_date_time -lboost_regex -lssl -lcrypto \
		-lmysqlclient
		INC_PARM=$(foreach d, $(INC_DIR), -I$d)

src/DMDBVerifier.cpp

+2 −1

Original line number	Diff line number	Diff line
		@@ -97,6 +97,7 @@ void DMDBVerifier::testInstrumentMapping(Instrument::SP instrument_sp)
		testColumnDefinition(columnTuple_vector,"file_name","varchar", true);
		testColumnDefinition(columnTuple_vector,"checksum","varchar", false);
		testColumnDefinition(columnTuple_vector,"checksum_gz","varchar", false);
		testColumnDefinition(columnTuple_vector,"dataunitsum","varchar", false);
		testColumnDefinition(columnTuple_vector,"update_time","timestamp", true);

		//Check mapping columns in table definition

src/WorkerThread.cpp

+13 −3

Original line number	Diff line number	Diff line
		@@ -19,6 +19,8 @@
		#include <soci/rowset.h>
		#include <soci/boost-optional.h>

		#include "dataunitsum.h"

		namespace FitsImporter_ns
		{

		@@ -387,6 +389,9 @@ void WorkerThread::processFits(boost::filesystem::path& origPath,
		//Compute checksum
		std::string checksum = computeMd5(origPath);

		//Compute dataunitsum
		std::string dataunitsum = compute_dataunitsum(origPath.string());

		//Search if file is already archived
		int duplicateMax = findDuplicateMax(session_sp, destination_sp, origPath, checksum);

		@@ -412,8 +417,8 @@ void WorkerThread::processFits(boost::filesystem::path& origPath,
		if(duplicateMax!=-1)
		{
		//Create metadata query
		std::string query = composeQuery(destination_sp, date, duplicateMax, checksum, checksum_gz,
		origPath, fitsFile_sp, readHDUSet);
		std::string query = composeQuery(destination_sp, date, duplicateMax, checksum, dataunitsum,
		checksum_gz, origPath, fitsFile_sp, readHDUSet);

		try
		{
		@@ -497,7 +502,8 @@ int WorkerThread::findDuplicateMax(ConnectionManager::SessionSP session_sp,
		// WorkerThread::composeQuery()
		//==============================================================================
		std::string WorkerThread::composeQuery(Destination::SP destination_sp,
		boost::gregorian::date& date, int duplicateMax, std::string checksum, std::string checksum_gz,
		boost::gregorian::date& date, int duplicateMax, std::string checksum,
		std::string dataunitsum, std::string checksum_gz,
		boost::filesystem::path& origPath, boost::shared_ptr<CCfits::FITS> fitsFile_sp, std::set<int>& readHDUSet)
		throw(CCfits::FitsException, std::runtime_error)
		{
		@@ -520,6 +526,10 @@ std::string WorkerThread::composeQuery(Destination::SP destination_sp,
		insertQuery << "," << destination_sp->getTable() << ".checksum";
		valuesQuery << ",\'" << checksum << "\'";

		//Append checksum
		insertQuery << "," << destination_sp->getTable() << ".dataunitsum";
		valuesQuery << ",\'" << dataunitsum << "\'";

		//Append checksum_gz
		insertQuery << "," << destination_sp->getTable() << ".checksum_gz";
		valuesQuery << ",\'" << checksum_gz << "\'";

src/WorkerThread.h

+2 −1

Original line number	Diff line number	Diff line
		@@ -75,7 +75,8 @@ protected:
		// [Protected] Metadata archiving methods
		//------------------------------------------------------------------------------
		virtual std::string composeQuery(Destination::SP, boost::gregorian::date&,
		int, std::string, std::string, boost::filesystem::path&, boost::shared_ptr<CCfits::FITS>,
		int, std::string, std::string, std::string, boost::filesystem::path&,
		boost::shared_ptr<CCfits::FITS>,
		std::set<int>&) throw(CCfits::FitsException, std::runtime_error);

		virtual void fillQueryFromFits(std::stringstream&, std::stringstream&,

src/main.cpp

+15 −4

Original line number	Diff line number	Diff line
		@@ -40,12 +40,23 @@
		#include <tango.h>
		#include <Version.h>

		#include "dataunitsum.h"

		int main(int argc,char *argv[])
		{
		if (std::string(argv[1]) == "--version"){
		if (std::string(argv[1]).compare("--version") == 0)
		{
		cout << version << endl;
		return(0);
		}
		else if ((argc ==3) && (std::string(argv[1]).compare("--dataunitsum") == 0))
		{
		cout << compute_dataunitsum(std::string(argv[2])) << endl;
		return(0);
		}



		try
		{
		// Initialise the device server