Use a VOSpace job ID in place of a UUID in file list names + set VOSpace... (4f6190c7) · Commits · VOSpace INAF / vospace-transfer-service

transfer_service/retrieve_executor.py

+2 −2

Original line number	Diff line number	Diff line
		@@ -255,7 +255,7 @@ class RetrieveExecutor(TaskExecutor):
		# if the storage type is 'cold'
		if self.storageType == "cold":
		self.tapeClient.connect()
		self.tapeClient.recall([ f["fullPath"] for f in blockFileList ])
		self.tapeClient.recall([ f["fullPath"] for f in blockFileList ], self.jobId)
		self.tapeClient.disconnect()

		# Loop on files in a block
		@@ -294,7 +294,7 @@ class RetrieveExecutor(TaskExecutor):
		# is 'cold'
		if self.storageType == "cold":
		self.tapeClient.connect()
		self.tapeClient.migrate([ f["fullPath"] for f in blockFileList ], self.tapePool)
		self.tapeClient.migrate([ f["fullPath"] for f in blockFileList ], self.tapePool, self.jobId)
		self.tapeClient.disconnect()

		blockFileList.clear()

transfer_service/tape_client.py

+17 −13

Original line number	Diff line number	Diff line
		@@ -11,7 +11,6 @@ import os
		import paramiko
		import scp
		import sys
		import uuid

		from config import Config
		from exceptions import ScpInvalidFileException
		@@ -24,6 +23,8 @@ from tape_task import TapeTask
		class TapeClient(object):
		# 'eeadm' command location on the tape library frontend
		EEADM = "/opt/ibm/ltfsee/bin/eeadm"
		# destination for the files containing the lists of files to recall or migrate
		VOSPACE_WD = "/tmp/vospace"

		def __init__(self, host, port, user, keyFile, logger):
		self.host = host
		@@ -144,24 +145,26 @@ class TapeClient(object):
		finally:
		self.scp.close()

		def migrate(self, fileList, tapePool):
		def migrate(self, fileList, tapePool, jobId):
		"""
		Migrates to tape all files whose absolute path is
		contained in 'fileList'.
		A tape pool and a VOSpace jobId are also required
		as parameters.
		"""
		self.logger.info("Starting MIGRATE operation...")
		tmp = str(uuid.uuid1().hex) + "-vos_migrate.tmp"
		migrateFileList = f"vos_migrate-{jobId}.lst"
		try:
		fp = open(tmp, "a")
		fp = open(migrateFileList, "a")
		except IOError:
		raise
		else:
		for f in fileList:
		fp.write(f"{f}\n")
		fp.close()
		self.copy(f"./{tmp}", f"/tmp/{tmp}")
		os.remove(f"./{tmp}")
		cmd = f"{self.EEADM} migrate /tmp/{tmp} -p {tapePool}"
		self.copy(f"./{migrateFileList}", f"{self.VOSPACE_WD}/{migrateFileList}")
		os.remove(f"./{migrateFileList}")
		cmd = f"{self.EEADM} migrate {self.VOSPACE_WD}/{migrateFileList} -p {tapePool}"
		try:
		stdin, stdout, stderr = self.client.exec_command(cmd)
		except Exception:
		@@ -176,24 +179,25 @@ class TapeClient(object):
		raise TapeClientException(cmd, exitCode, stderr)
		return exitCode

		def recall(self, fileList):
		def recall(self, fileList, jobId):
		"""
		Recalls from tape all files whose absolute path is
		contained in 'fileList'.
		A VOSpace job ID is also required as parameter.
		"""
		self.logger.info("Starting RECALL operation...")
		tmp = str(uuid.uuid1().hex) + "-vos_recall.tmp"
		recallFileList = f"vos_recall-{jobId}.lst"
		try:
		fp = open(tmp, "a")
		fp = open(recallFileList, "a")
		except IOError:
		raise
		else:
		for f in fileList:
		fp.write(f"{f}\n")
		fp.close()
		self.copy(f"./{tmp}", f"/tmp/{tmp}")
		os.remove(f"./{tmp}")
		cmd = f"{self.EEADM} recall /tmp/{tmp}"
		self.copy(f"./{recallFileList}", f"{self.VOSPACE_WD}/{recallFileList}")
		os.remove(f"./{recallFileList}")
		cmd = f"{self.EEADM} recall {self.VOSPACE_WD}/{recallFileList}"
		try:
		stdin, stdout, stderr = self.client.exec_command(cmd)
		except Exception: