Commit 3433be47 authored by Cristiano Urban's avatar Cristiano Urban
Browse files

Removed 'scan()' and 'scanRecursive()' methods. 



Signed-off-by: default avatarCristiano Urban <cristiano.urban@inaf.it>
parent 9b430741

transfer_service/store_preprocessor.py

+5 −33
Original line number Diff line number Diff line
@@ -15,6 +15,7 @@ from checksum import Checksum 
from file_grouper import FileGrouper

from db_connector import DbConnector

from node import Node

from system_utils import SystemUtils

from task_executor import TaskExecutor

from config import Config
@@ -22,6 +23,7 @@ from config import Config 
class StorePreprocessor(TaskExecutor):



    def __init__(self):

        self.systemUtils = SystemUtils()

        self.md5calc = Checksum()

        config = Config("vos_ts.conf")

        self.params = config.loadSection("file_grouper")
@@ -43,36 +45,6 @@ class StorePreprocessor(TaskExecutor): 
        self.nodeList = []

        super(StorePreprocessor, self).__init__()



    # Scan is performed only on the first level!

    def scan(self):

        dirList = []

        fileList = []

        elementList = os.listdir(self.path)

        for el in elementList:

            elPath = self.path + '/' + el

            if os.path.isdir(elPath):

                dirList.append(el)

            elif os.path.isfile(elPath):

                fileList.append(el)

            else:

                sys.exit("FATAL: invalid file/dir.")

        return [ dirList, fileList ]



    def scanRecursive(self):

        dirList = []

        fileList = []

        for folder, subfolders, files in os.walk(self.path, topdown = True):

            cwd = os.path.basename(folder)

            if folder != self.path:

                parent = os.path.dirname(folder)

                dirList.append(parent + '/' + cwd)

                i = 0

                for f in files:

                    files[i] = parent + '/' + cwd + '/' + f

                    i += 1

                fileList.append(files)

        return [ dirList, fileList ]



    def prepare(self, username):

        self.username = username

        self.path = "/home/" + username + "/store"
@@ -88,14 +60,14 @@ class StorePreprocessor(TaskExecutor): 


    def execute(self):

        # First scan to find crowded dirs

        [ dirs, files ] = self.scan()

        [ dirs, files ] = self.systemUtils.scan(self.path)



        # Create a .tar for all dirs matching the constraints, if any

        for dir in dirs:

            self.fileGrouper.recursive(self.path + '/' + dir)



        # Second scan after file grouper execution

        [ dirs, files ] = self.scan()

        [ dirs, files ] = self.systemUtils.scan(self.path)

        timestamp = dt.now().strftime("%Y_%m_%d-%H_%M_%S")



        # Case 1: /home/user/store contains both files and dirs
@@ -133,7 +105,7 @@ class StorePreprocessor(TaskExecutor): 
            sys.exit("The 'store' directory is empty.")



        # Third scan after directory structure 'check & repair'

        [ dirs, files ] = self.scanRecursive()

        [ dirs, files ] = self.systemUtils.scanRecursive(self.path)



        # File catalog update

        out = open("store_preprocessor_log.txt", "a")