Added test for file_catalog insert section + modified .gitignore. (edb0db71) · Commits · VOSpace INAF / vospace-transfer-service

.gitignore

+2 −0

Original line number	Diff line number	Diff line
		#####
		.directory
		file_catalog/06-setup.sh
		file_catalog/TODO.txt

transfer_service/db_connector.py

0 → 100644

+44 −0

Original line number	Diff line number	Diff line
		import psycopg2
		import sys

		from node import Node

		class DbConnector(object):

		def __init__(self, user, password, host, port, dbname):
		self.user = user
		self.password = password
		self.host = host
		self.port = port
		self.dbname = dbname

		def connect(self):
		try:
		self.conn = psycopg2.connect(user = self.user,
		password = self.password,
		host = self.host,
		port = self.port,
		database = self.dbname)
		except(Exception, psycopg2.Error) as error :
		sys.exit(f"Error while connecting to PostgreSQL: {error}")
		self.cursor = self.conn.cursor()

		def disconnect(self):
		if(self.conn):
		self.cursor.close()
		self.conn.close()

		def insertNode(self, node, parentOSPath):
		if(self.conn):
		print(f"parentOSPath: {parentOSPath}")
		self.cursor.execute("SELECT get_ltree_path(%s)", (parentOSPath,))
		self.conn.commit()
		parentLtreePath = self.cursor.fetchall()
		print(f"parentLtreePath: {parentLtreePath}")
		#self.cursor.execute("INSERT INTO Node(parentPath, name, type, ownerID, creatorID) VALUES (%s, %s, %s, %s, %s)",
		# (parentLtreePath, node.name, node.type, node.ownerID, node.creatorID,))
		#self.conn.commit()

		def selectNode(self):
		pass

transfer_service/node.py

0 → 100644

+88 −0

Original line number	Diff line number	Diff line
		import sys

		from datetime import datetime as dt


		class Node(object):

		def __init__(self, name, type):
		self.parentPath = None
		self.name = name
		self.type = type
		self.format = None
		self.groupRead = []
		self.groupWrite = []
		self.visibility = False
		self.delta = None
		self.contentType = None
		self.contentEncoding = None
		self.contentLength = None
		self.contentMD5 = None
		self.acceptViews = []
		self.provideViews = []
		self.protocols = []
		# used only in case of update
		self.lastModified = dt.now().strftime("%Y-%m-%d %H:%M:%S")

		self.types = [ "container", "data", "link" ]

		"""
		Getters
		"""




		"""
		Setters
		"""

		def setParentPath(self, parentPath):
		self.parentPath = parentPath

		def setName(self, name):
		self.name = name

		def setType(self, type):
		if type in self.types:
		self.type = type
		else:
		sys.exit(f"FATAL: Invalid type {type}")

		def setFormat(self, format):
		self.format = format

		# 'asyncTrans' flag
		def setTransferType(self, transferType):
		self.transferType = transferType

		def setBusyState(self, busyState):
		self.busyState = busyState

		def setOwnerID(self, ownerID):
		self.ownerID = ownerID

		def setCreatorID(self, creatorID):
		self.creatorID = creatorID

		def setGroupRead(self, groupRead):
		self.groupRead = groupRead

		def setGroupWrite(self, groupWrite):
		self.groupWrite = groupWrite

		# 'isPubilc' flag
		def setVisibility(self, visibility):
		self.visibility = visibility

		def setContentType(self, contentType):
		self.contentType = contentType

		def setContendEncoding(self, contentEncoding):
		self.contentEncoding = contentEncoding

		def setContentLength(self, contentLength):
		self.contentLength = contentLength

		def setContentMD5(self, contentMD5):
		self.contentMD5 = contentMD5

transfer_service/store_preprocessor.py

+45 −3

Original line number	Diff line number	Diff line
		@@ -10,6 +10,8 @@ import sys
		from datetime import datetime as dt
		from checksum import Checksum
		from file_grouper import FileGrouper
		from db_connector import DbConnector
		from node import Node


		class StorePreprocessor(object):
		@@ -17,7 +19,10 @@ class StorePreprocessor(object):
		def __init__(self):
		self.md5calc = Checksum()
		self.fileGrouper = FileGrouper(1000, 100 * (2 ** 30))
		self.dbConn = DbConnector("postgres", "postgres", "file_catalog", 5432, "vospace_testdb")
		self.dbConn.connect()

		# Scan is performed only on the first level!
		def scan(self):
		dirList = []
		fileList = []
		@@ -32,6 +37,20 @@ class StorePreprocessor(object):
		sys.exit("FATAL: invalid file/dir.")
		return [ dirList, fileList ]

		def scanRecursive(self):
		dirList = []
		fileList = []
		for folder, subfolders, files in os.walk(self.path, topdown = True):
		cwd = os.path.basename(folder)
		parent = os.path.dirname(folder)
		dirList.append(parent + '/' + cwd)
		i = 0
		for f in files:
		files[i] = parent + '/' + cwd + '/' + f
		i += 1
		fileList.append(files)
		return [ dirList, fileList ]

		def prepare(self, username):
		self.username = username
		self.path = "/home/" + username + "/store"
		@@ -46,14 +65,14 @@ class StorePreprocessor(object):
		os.chmod(os.path.join(folder, f), 0o555)

		def start(self):
		# First scan
		# First scan to find crowded dirs
		[ dirs, files ] = self.scan()

		# Create a .tar for all dirs matching the constraints, if any
		for dir in dirs:
		self.fileGrouper.recursive(self.path + '/' + dir)

		# Second scan
		# Second scan after file grouper execution
		[ dirs, files ] = self.scan()
		timestamp = dt.now().strftime("%Y_%m_%d-%H_%M_%S")

		@@ -91,7 +110,30 @@ class StorePreprocessor(object):
		else:
		sys.exit("The 'store' directory is empty.")

		# Third scan after directory structure 'check & repair'
		[ dirs, files ] = self.scanRecursive()

		# File catalog update
		for dir in dirs:
		print(f"DIR: {dir}")
		cnode = Node(os.path.basename(dir), "container");
		cnode.setParentPath(os.path.dirname(dir))
		cnode.setOwnerID("3354")
		cnode.setCreatorID("3354")
		self.dbConn.insertNode(cnode, cnode.parentPath)

		for flist in files:
		for file in flist:
		print(f"FILE {file}")
		dnode = Node(os.path.basename(file), "data")
		dnode.setParentPath(os.path.dirname(file))
		dnode.setOwnerID("3354")
		dnode.setCreatorID("3354")
		self.dbConn.insertNode(dnode, dnode.parentPath)

		self.dbConn.disconnect()

		# Test
		sp = StorePreprocessor()
		sp.prepare("cristiano")
		sp.prepare("curban")
		sp.start()
		No newline at end of file