Added size convertion from human readable to bytes + minor changes. (f5bce7af) · Commits · VOSpace INAF / vospace-transfer-service

transfer_service/checksum.py

+3 −1

Original line number	Diff line number	Diff line
		@@ -5,14 +5,16 @@ import os
		import sys

		from config import Config
		from system_utils import SystemUtils


		class Checksum(object):

		def __init__(self):
		self.systemUtils = SystemUtils()
		config = Config("/etc/vos_ts/vos_ts.conf")
		self.params = config.loadSection("checksum")
		self.fileBufferSize = self.params.getint("file_buffer_size")
		self.fileBufferSize = self.systemUtils.convertSizeToBytes(self.params["file_buffer_size"])
		self.md5FileSuffix = self.params["md5_file_suffix"]

		def setFileBufferSize(fileBufferSize):

+8 −3

Original line number	Diff line number	Diff line
		@@ -51,13 +51,13 @@ pkey_file_path = /root/.ssh/tape_rsa
		; suffix for files containing MD5 checksums
		md5_file_suffix = -md5sum.txt
		; buffer size in bytes when reading a chunk of data, default is 4096 B
		file_buffer_size = 4096
		file_buffer_size = 4096 B

		[file_grouper]
		; minimum number of files contained by a 'leaf', default is 1000
		min_num_files = 1000
		; maximum size in GB for a 'leaf', default is 100 GB
		max_dir_size = 100
		; maximum size for a 'leaf', default is 100 GB
		max_dir_size = 100 GB

		[scheduling]
		; maximum number of jobs within a 'pending' queue, default is 32
		@@ -70,6 +70,10 @@ max_terminated_jobs = 8
		; job queues (do NOT set to zero), default is 15 s
		exec_wait_time = 15

		[transfer]
		; split data to be retrieved in blocks of a given size
		block_size = 2 MB

		[cleanup]
		; Physically delete from disk all nodes previously deleted by the user via ui,
		; that are older than a given amount of time specified here below
		@@ -82,6 +86,7 @@ seconds = 30
		smtp_server = smtp2.oats.inaf.it
		smtp_port = 25
		no_reply_email = noreply-vospace@inaf.it
		admin_email = cristiano.urban@inaf.it


		####################

+5 −2

Original line number	Diff line number	Diff line
		@@ -8,6 +8,7 @@ import sys
		from checksum import Checksum
		from config import Config
		from db_connector import DbConnector
		from system_utils import SystemUtils
		from tape_client import TapeClient
		from task_executor import TaskExecutor

		@@ -15,6 +16,7 @@ from task_executor import TaskExecutor
		class RetrieveExecutor(TaskExecutor):

		def __init__(self):
		self.systemUtils = SystemUtils()
		config = Config("/etc/vos_ts/vos_ts.conf")
		self.params = config.loadSection("spectrum_archive")
		self.tapeClient = TapeClient(self.params["host"],
		@@ -31,13 +33,14 @@ class RetrieveExecutor(TaskExecutor):
		self.params["db"],
		1,
		1)
		self.params = config.loadSection("transfer")
		self.maxBlockSize = self.systemUtils.convertSizeToBytes(self.params["block_size"])
		self.storageType = None
		self.jobObj = None
		self.jobId = None
		self.nodeList = []
		self.fileList = []
		self.numBlocks = None
		self.maxBlockSize = 1536000
		super(RetrieveExecutor, self).__init__()

		def buildFileList(self):

+1 −1

Original line number	Diff line number	Diff line
		@@ -30,7 +30,7 @@ class StorePreprocessor(TaskExecutor):
		config = Config("/etc/vos_ts/vos_ts.conf")
		self.params = config.loadSection("file_grouper")
		self.fileGrouper = FileGrouper(self.params.getint("min_num_files"),
		self.params.getint("max_dir_size") * (2 ** 30))
		self.systemUtils.convertSizeToBytes(params["max_dir_size"]))
		self.params = config.loadSection("file_catalog")
		self.dbConn = DbConnector(self.params["user"],
		self.params["password"],

+22 −0

Original line number	Diff line number	Diff line
		@@ -6,6 +6,15 @@ import sys

		class SystemUtils(object):

		UNITS = {
		"B": 1,
		"KB": 1 << 10,
		"MB": 1 << 20,
		"GB": 1 << 30,
		"TB": 1 << 40,
		"PB": 1 << 50,
		}

		def __init__(self):
		pass

		@@ -67,7 +76,20 @@ class SystemUtils(object):
		sys.exit("FATAL: invalid file/dir.")
		return [ dirList, fileList ]

		def convertSizeToBytes(self, sizeStr):
		size = int(sizeStr.split()[0])
		unit = sizeStr.split()[1]
		if unit not in self.UNITS:
		return False
		else:
		return size * self.UNITS[unit]





		# Test
		#sysUtils = SystemUtils()
		#info = sysUtils.userInfo("cristiano")
		#print(sysUtils.convertSizeInBytes(" 10 MB "))
		#print(info)