Merge branch 'testing' (995fdedf) · Commits · VOSpace INAF / vospace-transfer-service

file_catalog/init.sql

+26 −17

Original line number	Diff line number	Diff line
		@@ -34,26 +34,26 @@ CREATE TYPE NodeType AS ENUM ('container', 'data', 'link');
		CREATE TABLE Node (
		nodeID BIGSERIAL NOT NULL,
		path LTREE default NULL,
		name VARCHAR(256) NOT NULL,
		name VARCHAR NOT NULL,
		type NodeType NOT NULL,
		format VARCHAR default NULL,
		/* format serve per distinguere unstuctured (format=NULL) da structured che hanno un formato noto */
		asyncTrans BOOLEAN NOT NULL,
		/* asyncTransf serve per indicare se il nodo e` ospitato da un cold storage e deve essere necessariamente
		trasferito con un trasferimento asincrono */
		busyState CHAR(1) NOT NULL,
		ownerID NUMERIC(40) NOT NULL,
		creatorID NUMERIC(40) NOT NULL,
		groupRead VARCHAR(256) default NULL,
		groupWrite VARCHAR(256) default NULL,
		busyState BOOLEAN NOT NULL,
		ownerID VARCHAR NOT NULL,
		creatorID VARCHAR NOT NULL,
		groupRead VARCHAR[] default NULL,
		groupWrite VARCHAR[] default NULL,
		isPublic BOOLEAN NOT NULL,
		delta NUMERIC(20) default NULL,
		delta BIGINT default NULL,
		/* potrebbe essere un delta di dati trasferito durante un trasferimento asincrono.
		Dovrebbe stare sul servizio che fa il trasferimento (es. redis) */
		contentType VARCHAR(100) default NULL,
		contentEncoding VARCHAR(50) default NULL,
		contentLength NUMERIC(20) default NULL,
		contentMD5 BYTEA default NULL,
		contentType VARCHAR default NULL,
		contentEncoding VARCHAR default NULL,
		contentLength BIGINT default NULL,
		contentMD5 TEXT default NULL,
		createdOn TIMESTAMP default CURRENT_TIMESTAMP,
		lastModified TIMESTAMP NOT NULL,
		/*
		@@ -62,7 +62,7 @@ Dovrebbe stare sul servizio che fa il trasferimento (es. redis) */
		acceptViews TEXT[] default NULL,
		provideViews TEXT[] default NULL,
		/*
		storageID VARCHAR(256),
		storageID VARCHAR,
		serve per mappare il nome del servizio di storage da interrogare per accedere al contenuto di questo nodo
		*/
		protocols TEXT[] default NULL,
		@@ -72,17 +72,26 @@ Dovrebbe stare sul servizio che fa il trasferimento (es. redis) */

		CREATE TABLE NodeProperty (
		nodeID BIGSERIAL,
		propertyURI VARCHAR(256) NOT NULL,
		propertyValue VARCHAR(512) default NULL,
		propertyURI VARCHAR NOT NULL,
		propertyValue VARCHAR default NULL,
		lastModified TIMESTAMP default CURRENT_TIMESTAMP,
		-- support replication with a fake primary key
		_rep_support NUMERIC(20) NOT NULL PRIMARY KEY,
		-- _rep_support BIGINT NOT NULL PRIMARY KEY,
		foreign key (nodeID) references Node (nodeID)
		);


		CREATE TABLE DeletedNode
		(
		nodeID BIGSERIAL NOT NULL,
		name VARCHAR NOT NULL,
		ownerID VARCHAR NOT NULL,
		lastModified TIMESTAMP default CURRENT_TIMESTAMP,
		PRIMARY KEY (nodeID)
		);

		/*
		Initialize root node for vospace
		*/

		insert into Node (name, type, asyncTrans, busyState, ownerID, creatorID, groupRead, groupWrite, isPublic, lastModified)
		values ('YOUR_ROOT_NODE', 'container', '1', 'N', '9127391732918723981732198273275643832902', '9127391732918723981732198273275643832902', 'YOUR_ADMIN_GROUP', 'YOUR_ADMIN_GROUP', '1', CURRENT_TIMESTAMP);
		insert into Node (name, type, asyncTrans, busyState, ownerID, creatorID, groupRead, groupWrite, isPublic, lastModified) values ('YOUR_ROOT_NODE', 'container', '1', 'N', '9127391732918723981732198273275643832902', '9127391732918723981732198273275643832902', '{"YOUR_ADMIN_GROUP"}', '{"YOUR_ADMIN_GROUP"}', '1', CURRENT_TIMESTAMP);
		No newline at end of file

tests/checksum.py

0 → 100644

+29 −0

Original line number	Diff line number	Diff line
		import hashlib
		import sys
		import os

		script, dir = sys.argv

		def md5sum(fname):
		hash_md5 = hashlib.md5()
		with open(fname, "rb") as f:
		for chunk in iter(lambda: f.read(4096), b""):
		hash_md5.update(chunk)
		return hash_md5.hexdigest()

		for folder, subfolder, files in os.walk(dir, topdown = False):
		#print(folder, os.path.basename(folder), subfolder, files)
		cwd = os.path.basename(folder)
		parent = os.path.dirname(folder)
		#parent = os.path.basename(parent)
		#print(cwd, parent, os.path.abspath(parent))
		#print(os.path.abspath(folder))
		if not os.path.exists(os.path.abspath(parent) + '/' + cwd + "-md5sum.txt"):
		#if not os.listdir(folder) == []:
		if not files == []:
		md5file = open(os.path.abspath(parent) + '/' + cwd + "-md5sum.txt", 'w')
		for file in files:
		#print(file)
		md5file.write(md5sum(os.path.abspath(folder) + '/' + file) + " ./" + file)
		md5file.write('\n')
		md5file.close()

transfer_service/checksum.py

0 → 100644

+38 −0

Original line number	Diff line number	Diff line
		import hashlib
		import os
		import sys


		class Checksum(object):

		def __init__(self):
		self.fileBufferSize = 4096
		self.md5FileSuffix = "-md5sum.txt"

		def setFileBufferSize(fileBufferSize):
		self.fileBufferSize = fileBufferSize

		def md5sum(self, filePath):
		md5Hash = hashlib.md5()
		with open(filePath, "rb") as f:
		for chunk in iter(lambda: f.read(self.fileBufferSize), b""):
		md5Hash.update(chunk)
		return md5Hash.hexdigest()

		def recursive(self, folderPath):
		for folder, subfolder, files in os.walk(folderPath, topdown = False):
		cwd = os.path.basename(folder)
		parent = os.path.dirname(folder)
		md5FilePath = os.path.abspath(parent) + '/' + cwd + self.md5FileSuffix
		if not os.path.exists(md5FilePath):
		if not files == []:
		md5file = open(md5FilePath, 'w')
		for file in files:
		filePath = os.path.abspath(folder) + '/' + file
		md5file.write(self.md5sum(filePath) + " ./" + file + '\n')
		md5file.close()


		script, folderName = sys.argv
		md5calc = Checksum()
		md5calc.recursive(folderName)