Loading transfer_service/retrieve_cleaner.py +17 −10 Original line number Diff line number Diff line Loading @@ -86,11 +86,13 @@ class RetrieveCleaner(TaskExecutor): self.logger.exception("FATAL: unable to obtain info about VOSpace nodes.") return False numNodes = len(self.nodeList) numDestPaths = len(self.destPathList) self.logger.info(f"Number of VOSpace nodes involved: {numNodes}") while numNodes > 0: self.logger.info(f"Number of associated physical paths: {numDestPaths}") while numNodes > 0 and numDestPaths > 0 and numNodes == numDestPaths: time.sleep(0.2) vospacePath = self.nodeList[numNodes - 1] destPath = self.destPathList[numNodes - 1] destPath = self.destPathList[numDestPaths - 1] try: busy = self.dbConn.nodeIsBusy(vospacePath) except Exception: Loading @@ -112,7 +114,7 @@ class RetrieveCleaner(TaskExecutor): except FileNotFoundError: self.logger.exception(f"Cannot find '{destPath}', skip...") # check for empty dirs and remove them # Check for empty dirs and remove them basePath = self.storageRetrievePath.replace("{username}", self.username) for root, dirs, files in os.walk(basePath, topdown = False): for dir in dirs: Loading @@ -120,8 +122,9 @@ class RetrieveCleaner(TaskExecutor): if not os.listdir(dirPath): os.rmdir(dirPath) self.nodeList.pop(numNodes - 1) self.destPathList.pop(numNodes - 1) self.destPathList.pop(numDestPaths - 1) numNodes -= 1 numDestPaths -= 1 except Exception: self.logger.exception("FATAL: something went wrong while cleaning the expired data.") return False Loading @@ -148,6 +151,10 @@ class RetrieveCleaner(TaskExecutor): if srcQueueLen > 0: self.jobObj = self.srcQueue.getJob() self.nodeList = self.jobObj.nodeList.copy() # The 'destPathList' key may not be present (e.g. when a user performs a login # through VOSpace UI and launches an async recall job, but he/she is not present # in the 'user' table of the VOSpace database) if "destPathList" in self.jobObj.jobInfo: self.destPathList = self.jobObj.jobInfo["destPathList"].copy() if self.dataHasExpired(): if self.execute(): Loading Loading
transfer_service/retrieve_cleaner.py +17 −10 Original line number Diff line number Diff line Loading @@ -86,11 +86,13 @@ class RetrieveCleaner(TaskExecutor): self.logger.exception("FATAL: unable to obtain info about VOSpace nodes.") return False numNodes = len(self.nodeList) numDestPaths = len(self.destPathList) self.logger.info(f"Number of VOSpace nodes involved: {numNodes}") while numNodes > 0: self.logger.info(f"Number of associated physical paths: {numDestPaths}") while numNodes > 0 and numDestPaths > 0 and numNodes == numDestPaths: time.sleep(0.2) vospacePath = self.nodeList[numNodes - 1] destPath = self.destPathList[numNodes - 1] destPath = self.destPathList[numDestPaths - 1] try: busy = self.dbConn.nodeIsBusy(vospacePath) except Exception: Loading @@ -112,7 +114,7 @@ class RetrieveCleaner(TaskExecutor): except FileNotFoundError: self.logger.exception(f"Cannot find '{destPath}', skip...") # check for empty dirs and remove them # Check for empty dirs and remove them basePath = self.storageRetrievePath.replace("{username}", self.username) for root, dirs, files in os.walk(basePath, topdown = False): for dir in dirs: Loading @@ -120,8 +122,9 @@ class RetrieveCleaner(TaskExecutor): if not os.listdir(dirPath): os.rmdir(dirPath) self.nodeList.pop(numNodes - 1) self.destPathList.pop(numNodes - 1) self.destPathList.pop(numDestPaths - 1) numNodes -= 1 numDestPaths -= 1 except Exception: self.logger.exception("FATAL: something went wrong while cleaning the expired data.") return False Loading @@ -148,6 +151,10 @@ class RetrieveCleaner(TaskExecutor): if srcQueueLen > 0: self.jobObj = self.srcQueue.getJob() self.nodeList = self.jobObj.nodeList.copy() # The 'destPathList' key may not be present (e.g. when a user performs a login # through VOSpace UI and launches an async recall job, but he/she is not present # in the 'user' table of the VOSpace database) if "destPathList" in self.jobObj.jobInfo: self.destPathList = self.jobObj.jobInfo["destPathList"].copy() if self.dataHasExpired(): if self.execute(): Loading