Added basic logging + added check on threads and processes startup + minor changes. (24de010c) · Commits · VOSpace INAF / vospace-transfer-service

transfer_service/abort_job_rpc_server.py

+27 −8

Original line number	Diff line number	Diff line
		#!/usr/bin/env python

		import logging

		from redis_rpc_server import RedisRPCServer
		from db_connector import DbConnector
		from config import Config
		@@ -10,21 +12,38 @@ class AbortJobRPCServer(RedisRPCServer):
		def __init__(self, host, port, db, rpcQueue):
		self.type = "abort"
		config = Config("/etc/vos_ts/vos_ts.conf")
		self.params = config.loadSection("file_catalog")
		self.dbConn = DbConnector(self.params["user"],
		self.params["password"],
		self.params["host"],
		self.params.getint("port"),
		self.params["db"],
		params = config.loadSection("file_catalog")
		self.dbConn = DbConnector(params["user"],
		params["password"],
		params["host"],
		params.getint("port"),
		params["db"],
		1,
		2)
		params = config.loadSection("transfer_node")
		self.storageStorePath = params["store_path"]
		params = config.loadSection("scheduling")
		self.maxPendingJobs = params.getint("max_pending_jobs")
		params = config.loadSection("logging")
		self.logger = logging.getLogger("AbortJobRPCServer")
		logLevel = "logging." + params["log_level"]
		logDir = params["log_dir"]
		logFile = logDir + '/' + "abort_job_rpc_server.log"
		self.logger.setLevel(eval(logLevel))
		logFileHandler = logging.FileHandler(logFile)
		logStreamHandler = logging.StreamHandler()
		logFormatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
		logFileHandler.setFormatter(logFormatter)
		logStreamHandler.setFormatter(logFormatter)
		self.logger.addHandler(logFileHandler)
		self.logger.addHandler(logStreamHandler)
		super(AbortJobRPCServer, self).__init__(host, port, db, rpcQueue)

		def callback(self, requestBody):
		#TODO
		# do something...
		return 42
		pass

		def run(self):
		print(f"Starting RPC server of type {self.type}...")
		self.logger.info(f"Starting RPC server of type {self.type}...")
		super(AbortJobRPCServer, self).run()

transfer_service/checksum.py

+3 −3

Original line number	Diff line number	Diff line
		@@ -13,9 +13,9 @@ class Checksum(object):
		def __init__(self):
		self.systemUtils = SystemUtils()
		config = Config("/etc/vos_ts/vos_ts.conf")
		self.params = config.loadSection("checksum")
		self.fileBufferSize = self.systemUtils.convertSizeToBytes(self.params["file_buffer_size"])
		self.md5FileSuffix = self.params["md5_file_suffix"]
		params = config.loadSection("checksum")
		self.fileBufferSize = self.systemUtils.convertSizeToBytes(params["file_buffer_size"])
		self.md5FileSuffix = params["md5_file_suffix"]

		def setFileBufferSize(fileBufferSize):
		"""Sets the buffer size in bytes when reading a chunk of data."""

transfer_service/cli_handler.py

+7 −0

Original line number	Diff line number	Diff line
		#!/usr/bin/env python

		import sys
		import time

		from data_rpc_server import DataRPCServer
		from import_rpc_server import ImportRPCServer
		@@ -31,3 +32,9 @@ class CliHandler(object):
		def start(self):
		for srv in self.rpcServerList:
		srv.start()
		running = False
		while not running:
		time.sleep(1)
		running = True
		for srv in self.rpcServerList:
		running &= srv.is_alive()

transfer_service/data_rpc_server.py

+10 −10

Original line number	Diff line number	Diff line
		@@ -25,18 +25,18 @@ class DataRPCServer(RedisRPCServer):
		self.type = "data"
		self.storeAck = False
		config = Config("/etc/vos_ts/vos_ts.conf")
		self.params = config.loadSection("file_catalog")
		self.dbConn = DbConnector(self.params["user"],
		self.params["password"],
		self.params["host"],
		self.params.getint("port"),
		self.params["db"],
		params = config.loadSection("file_catalog")
		self.dbConn = DbConnector(params["user"],
		params["password"],
		params["host"],
		params.getint("port"),
		params["db"],
		1,
		2)
		self.params = config.loadSection("transfer_node")
		self.storageStorePath = self.params["store_path"]
		self.params = config.loadSection("scheduling")
		self.maxPendingJobs = self.params.getint("max_pending_jobs")
		params = config.loadSection("transfer_node")
		self.storageStorePath = params["store_path"]
		params = config.loadSection("scheduling")
		self.maxPendingJobs = params.getint("max_pending_jobs")
		params = config.loadSection("logging")
		self.logger = logging.getLogger("DataRPCServer")
		logLevel = "logging." + params["log_level"]

transfer_service/file_grouper.py

+0 −1

Original line number	Diff line number	Diff line
		@@ -20,7 +20,6 @@ class FileGrouper(object):
		"""Sets the 'minimum number of files' constraint."""
		self.minNumOfFiles = minNumOfFiles


		def setMaxFolderSize(self, maxFolderSize):
		"""Sets the 'maximum folder size' constraint."""
		self.maxFolderSize = maxFolderSize