Commit 6efbc7b0 authored by Cristiano Urban's avatar Cristiano Urban
Browse files

Consider only file sizes (apparent size). 



Signed-off-by: default avatarCristiano Urban <cristiano.urban@inaf.it>
parent 2d865f93

transfer_service/file_grouper.py

+1 −2
Original line number Diff line number Diff line
@@ -49,8 +49,7 @@ class FileGrouper(object): 
            files.append(os.path.abspath(folder) + '/' + f)  

        folderSize = 0

        for f in files:

            statinfo = os.stat(f)

            folderSize += statinfo.st_size           

            folderSize += os.path.getsize(f)

        return folderSize



    def getNumOfFiles(self, folder):

transfer_service/retrieve_executor.py

+4 −3
Original line number Diff line number Diff line
@@ -132,12 +132,13 @@ class RetrieveExecutor(TaskExecutor): 
                md5calc = Checksum()

                if os.path.isdir(srcPath) and not os.path.islink(srcPath):    

                    for root, dirs, files in os.walk(srcPath, topdown = False):

                        dirSize = os.stat(root).st_size

                        self.totalSize += dirSize

                        #dirSize = os.stat(root).st_size

                        #self.totalSize += dirSize

                        for f in files:

                            fullPath = os.path.join(root, f)

                            if md5calc.fileIsValid(fullPath) and not os.path.islink(fullPath):

                                fileSize = os.stat(fullPath).st_size

                                #fileSize = os.stat(fullPath).st_size

                                fileSize = os.path.getsize(fullPath)

                                fileInfo = {

                                               "baseSrcPath": baseSrcPath,

                                               "fullPath": fullPath,

transfer_service/store_preprocessor.py

+1 −1
Original line number Diff line number Diff line
@@ -227,7 +227,7 @@ class StorePreprocessor(TaskExecutor): 
                            self.logger.exception("FATAL: unable to update the file catalog.")

                            return False

            self.logger.info("Overall data size calculation")

            self.jobObj.jobInfo["dataSize"] = self.systemUtils.getSize(self.path) - os.stat(self.path).st_size

            self.jobObj.jobInfo["dataSize"] = self.systemUtils.getSize(self.path)

        except Exception:

            self.logger.exception("FATAL: something went wrong during the preprocessing phase.")

            return False

transfer_service/system_utils.py

+2 −3
Original line number Diff line number Diff line
@@ -117,18 +117,17 @@ class SystemUtils(object): 
        """

        size = 0

        if os.path.isfile(path) and not os.path.islink(path):

            size = os.stat(path).st_size

            size = os.path.getsize(path)

        elif os.path.isdir(path) and not os.path.islink(path):            

            for folder, subfolders, files in os.walk(path, topdown = True):

                cwd = os.path.basename(folder)

                parent = os.path.dirname(folder)

                base = parent + '/' + cwd

                if not os.path.islink(folder):

                    size += os.stat(folder).st_size

                    for f in files:

                        file = base + '/' + f

                        if not os.path.islink(file):

                            size += os.stat(file).st_size

                            size += os.path.getsize(file)

        return size



    def getFileSystemSize(self, path):