Commit 86c0cf81 authored by Cristiano Urban's avatar Cristiano Urban
Browse files

FileGrouper integration in StorePreprocessor class. 



Signed-off-by: default avatarCristiano Urban <cristiano.urban@inaf.it>
parent 36609172

transfer_service/file_grouper.py

+2 −2
Original line number Diff line number Diff line
@@ -44,7 +44,7 @@ class FileGrouper(object): 
        return len(os.listdir(folder))



    # Start the .tar creation procedure

    def start(self, folderPath):

    def recursive(self, folderPath):

        self.folderPath = folderPath

        leaves = self.getLeaves()

        for folder in leaves:
@@ -66,4 +66,4 @@ class FileGrouper(object): 
# Test (creates a .tar of all the leaves containing at least 2 files and having size lower than 100G,

# than removes those directories)

fg = FileGrouper(1, 100 * (2 ** 30))

fg.start("/home/cristiano/Lavoro/md5_test/store/a")
 
fg.recursive("/home/cristiano/Lavoro/md5_test/store/a")

transfer_service/store_preprocessor.py

+5 −0
Original line number Diff line number Diff line
@@ -9,12 +9,14 @@ import sys 


from datetime import datetime as dt

from checksum import Checksum

from file_grouper import FileGrouper





class StorePreprocessor(object):



    def __init__(self):

        self.md5calc = Checksum()

        self.fileGrouper = FileGrouper(1000, 100 * (2 ** 30))



    def scan(self):

        dirList = []
@@ -58,6 +60,7 @@ class StorePreprocessor(object): 
            for dir in dirs:

                srcPath = self.path + '/' + dir

                shutil.move(srcPath, destPath)

            self.fileGrouper.recursive(destPath)

            self.md5calc.recursive(destPath)

        elif files and not dirs:

            destPath = self.path + '/' + timestamp
@@ -68,10 +71,12 @@ class StorePreprocessor(object): 
            for file in files:

                srcPath = self.path + '/' + file

                shutil.move(srcPath, destPath)

            self.fileGrouper.recursive(destPath)

            self.md5calc.recursive(destPath)

        elif not files and dirs:

            for dir in dirs:

                destPath = self.path + '/' + dir

                self.fileGrouper.recursive(destPath)

                self.md5calc.recursive(destPath)

        else:

            sys.exit("The 'store' directory is empty.")