Loading transfer_service/file_grouper.py 0 → 100644 +59 −0 Original line number Diff line number Diff line import os import shutil import sys import subprocess class FileGrouper(object): # Constructor requiring the path of the folder, the minimum number of files # and the maximum folder size in bytes def __init__(self, folderPath, minNumOfFiles, maxFolderSize): self.folderPath = folderPath self.minNumOfFiles = minNumOfFiles self.maxFolderSize = maxFolderSize # Returns a list of all the 'leaves' (directories not # containing subdirectories) within the path def getLeaves(self): leaves = [] for root, dirs, files in os.walk(self.folderPath): if not dirs: leaves.append(root) return leaves # Returns the folder size in bytes def getFolderSize(self, folder): files = [ os.path.abspath(f) for f in os.listdir() ] folderSize = 0 for f in files: statinfo = os.stat(f) folderSize += statinfo.st_size return folderSize # Returns the number of files contained into a folder def getNumOfFiles(self, folder): return len(os.listdir(folder)) # Start the .tar creation procedure def start(self): leaves = self.getLeaves() for folder in leaves: if(self.getNumOfFiles(folder) > self.minNumOfFiles and self.getFolderSize(folder) < self.maxFolderSize): cwd = os.getcwd() parent = os.path.dirname(folder) os.chdir(parent) sp = subprocess.run(["tar", "-cf", os.path.basename(folder) + ".tar", os.path.basename(folder)], capture_output = True) if(sp.returncode or sp.stderr): sys.exit(f"Error: cannot create a .tar for {folder}") else: try: shutil.rmtree(folder) except OSError as e: print(f"Error: {folder} => {e.strerror}") os.chdir(cwd) # Test (creates a .tar of all the leaves containing at least 2 files and having size lower than 100G, # than removes that directories) fg = FileGrouper("/home/cristiano/Lavoro/md5_test/store/a", 1, 100 * (2 ** 30)) fg.start() No newline at end of file Loading
transfer_service/file_grouper.py 0 → 100644 +59 −0 Original line number Diff line number Diff line import os import shutil import sys import subprocess class FileGrouper(object): # Constructor requiring the path of the folder, the minimum number of files # and the maximum folder size in bytes def __init__(self, folderPath, minNumOfFiles, maxFolderSize): self.folderPath = folderPath self.minNumOfFiles = minNumOfFiles self.maxFolderSize = maxFolderSize # Returns a list of all the 'leaves' (directories not # containing subdirectories) within the path def getLeaves(self): leaves = [] for root, dirs, files in os.walk(self.folderPath): if not dirs: leaves.append(root) return leaves # Returns the folder size in bytes def getFolderSize(self, folder): files = [ os.path.abspath(f) for f in os.listdir() ] folderSize = 0 for f in files: statinfo = os.stat(f) folderSize += statinfo.st_size return folderSize # Returns the number of files contained into a folder def getNumOfFiles(self, folder): return len(os.listdir(folder)) # Start the .tar creation procedure def start(self): leaves = self.getLeaves() for folder in leaves: if(self.getNumOfFiles(folder) > self.minNumOfFiles and self.getFolderSize(folder) < self.maxFolderSize): cwd = os.getcwd() parent = os.path.dirname(folder) os.chdir(parent) sp = subprocess.run(["tar", "-cf", os.path.basename(folder) + ".tar", os.path.basename(folder)], capture_output = True) if(sp.returncode or sp.stderr): sys.exit(f"Error: cannot create a .tar for {folder}") else: try: shutil.rmtree(folder) except OSError as e: print(f"Error: {folder} => {e.strerror}") os.chdir(cwd) # Test (creates a .tar of all the leaves containing at least 2 files and having size lower than 100G, # than removes that directories) fg = FileGrouper("/home/cristiano/Lavoro/md5_test/store/a", 1, 100 * (2 ** 30)) fg.start() No newline at end of file