Merge branch 'testing' (84560517) · Commits · VOSpace INAF / vospace-transfer-service

base/Dockerfile

+10 −0

Original line number	Diff line number	Diff line
		@@ -7,6 +7,16 @@
		# Use CentOS 7 as base image
		FROM centos:7

		# Copy the repository file to the container
		COPY vault.repo /root/vault.repo

		# Update the system and install basic utilities
		RUN yum-config-manager --disable base && \
		yum-config-manager --disable updates && \
		yum-config-manager --disable extras && \
		cat /root/vault.repo >> /etc/yum.repos.d/Centos-Vault.repo && \
		yum clean all && yum makecache

		# Install epel repo
		RUN yum update -y && yum -y install epel-release

base/vault.repo

0 → 100644

+15 −0

Original line number	Diff line number	Diff line
		[Vault-base]
		name=Vault - CentOS-$releasever Base
		baseurl=http://vault.centos.org/centos/$releasever/os/$basearch/
		gpgcheck=0
		gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-$releasever
		[Vault-updates]
		name=Vault - CentOS-$releasever Updates
		baseurl=http://vault.centos.org/centos/$releasever/updates/$basearch/
		gpgcheck=0
		gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-$releasever
		[Vault-extras]
		name=Vault - CentOS-$releasever Extras
		baseurl=http://vault.centos.org/centos/$releasever/extras/$basearch/
		gpgcheck=0
		gpgkey=file:///etc/pki/rpm-gpg/RPM-GPG-KEY-CentOS-$releasever

client/vos_storage

+12 −12

Original line number	Diff line number	Diff line
		@@ -28,8 +28,8 @@ class VOSStorage(RedisRPCClient):
		storageType = None
		storageBasePath = None
		storageHostname = None
		tapePool = None
		tapePoolList = []
		tapeHSMFilesystem = None
		tapeHSMFilesystemList = []
		while storageType not in ("cold", "hot"):
		try:
		storageType = input("\nStorage type ['cold' or 'hot']: ")
		@@ -40,17 +40,17 @@ class VOSStorage(RedisRPCClient):
		except KeyboardInterrupt:
		sys.exit("\nCTRL+C detected. Exiting...")
		if storageType == "cold":
		storageRequest = { "requestType": "TAPE_POOL_LST" }
		storageRequest = { "requestType": "TAPE_HSM_FS_LST" }
		storageResponse = self.call(storageRequest)
		if "responseType" not in storageResponse:
		sys.exit("FATAL: Malformed response, storage acknowledge expected.\n")
		elif storageResponse["responseType"] == "TAPE_POOL_LST_DONE":
		tapePoolList = storageResponse["tapePoolList"]
		while tapePool not in tapePoolList:
		print("\nSelect one of the available tape pools:")
		print("\n" + tabulate([ [t] for t in tapePoolList ], headers = ["tape_pool"], tablefmt = "pretty") + "\n")
		elif storageResponse["responseType"] == "TAPE_HSM_FS_LST_DONE":
		tapeHSMFilesystemList = storageResponse["tapeHSMFilesystemList"]
		while tapeHSMFilesystem not in tapeHSMFilesystemList:
		print("\nSelect one of the available tape HSM filesystems:")
		print("\n" + tabulate([ [t] for t in tapeHSMFilesystemList ], headers = ["tape_hsm_fs"], tablefmt = "pretty") + "\n")
		try:
		tapePool = input("Please, insert a tape pool: ")
		tapeHSMFilesystem = input("Please, insert a tape HSM filesystem: ")
		except ValueError:
		print("Input type is not valid!")
		except EOFError:
		@@ -81,7 +81,7 @@ class VOSStorage(RedisRPCClient):
		"storageType": storageType,
		"basePath": storageBasePath,
		"hostname": storageHostname,
		"tapePool": tapePool }
		"tapeHSMFilesystem": tapeHSMFilesystem }
		storageResponse = self.call(storageRequest)

		if "responseType" not in storageResponse:
		@@ -202,8 +202,8 @@ DESCRIPTION
		Adding 'hot' or 'cold' storage points requires a base path
		to be specified (e.g. '/mnt/my_storage/users').

		Adding a 'cold' storage point requires selecting a tape pool
		(a tape pool list is provided).
		Adding a 'cold' storage point requires selecting a tape HSM
		filesystem (a tape HSM filesystem list is provided).

		All storage points require a valid hostname or IP address.
		""")

transfer_service/config/vos_ts.conf.sample

+2 −2

Original line number	Diff line number	Diff line
		@@ -30,8 +30,8 @@ port = 6379
		; db index representing the db that stores the scheduling queues, default is 0
		db_sched = 0

		# Spectrum Archive
		[spectrum_archive]
		# IBM Spectrum Protect
		[spectrum_protect]
		; hostname or IP address of the tape library frontend
		host = <tape_frontend_hostname>
		; SSH port

transfer_service/db_connector.py

+10 −10

Original line number	Diff line number	Diff line
		@@ -171,7 +171,7 @@ class DbConnector(object):
		conn = self.getConnection()
		cursor = conn.cursor(cursor_factory = RealDictCursor)
		cursor.execute("""
		SELECT storage_type, base_path, user_name, tstamp_wrapper_dir, '/' \|\| fs_path AS os_path, content_length, tape_pool
		SELECT storage_type, base_path, user_name, tstamp_wrapper_dir, '/' \|\| fs_path AS os_path, content_length, tape_hsm_fs
		FROM node n
		JOIN location l ON n.location_id = l.location_id
		JOIN storage s ON s.storage_id = l.storage_src_id
		@@ -192,7 +192,7 @@ class DbConnector(object):
		tstampWrappedDir = result[0]["tstamp_wrapper_dir"]
		osPath = result[0]["os_path"]
		contentLength = result[0]["content_length"]
		tapePool = result[0]["tape_pool"]
		tapeHSMFilesystem = result[0]["tape_hsm_fs"]
		if tstampWrappedDir is None:
		baseSrcPath = basePath + "/" + userName
		else:
		@@ -205,7 +205,7 @@ class DbConnector(object):
		"username": userName,
		"osPath": osPath,
		"contentLength": contentLength,
		"tapePool": tapePool
		"tapeHSMFilesystem": tapeHSMFilesystem
		}
		return fileInfo
		finally:
		@@ -824,12 +824,12 @@ class DbConnector(object):
		finally:
		self.connPool.putconn(conn, close = False)

		def getTapePool(self, storageId):
		"""Returns the tape pool for a given storage id, if the storage is cold. Otherwise it returns 'False'."""
		def getTapeHSMFilesystem(self, storageId):
		"""Returns the tape HSM filesystem for a given storage id, if the storage is cold. Otherwise it returns 'False'."""
		try:
		conn = self.getConnection()
		cursor = conn.cursor(cursor_factory = RealDictCursor)
		cursor.execute("SELECT tape_pool FROM storage WHERE storage_id = %s;", (storageId,))
		cursor.execute("SELECT tape_hsm_fs FROM storage WHERE storage_id = %s;", (storageId,))
		result = cursor.fetchall()
		cursor.close()
		except Exception:
		@@ -838,7 +838,7 @@ class DbConnector(object):
		raise
		else:
		if result:
		return result[0]["tape_pool"]
		return result[0]["tape_hsm_fs"]
		else:
		return False
		finally:
		@@ -1250,7 +1250,7 @@ class DbConnector(object):

		##### Storage #####

		def insertStorage(self, storageType, storageBasePath, storageHostname, vospaceUserBasePath, tapePool = None):
		def insertStorage(self, storageType, storageBasePath, storageHostname, vospaceUserBasePath, tapeHSMFilesystem = None):
		"""Inserts a storage point."""
		if not self.getStorageId(storageBasePath):
		try:
		@@ -1260,14 +1260,14 @@ class DbConnector(object):
		INSERT INTO storage(storage_type,
		base_path,
		hostname,
		tape_pool)
		tape_hsm_fs)
		VALUES (%s, %s, %s, %s)
		RETURNING storage_id;
		""",
		(storageType,
		storageBasePath,
		storageHostname,
		tapePool,))
		tapeHSMFilesystem,))
		storageSrcId = cursor.fetchall()[0]["storage_id"]
		except Exception:
		if not conn.closed: