Commit 9bfbbae5 authored by Cristiano Urban's avatar Cristiano Urban
Browse files

Added timestamp wrapped directory mechanism + adde files for testing purposes. 



Signed-off-by: default avatarCristiano Urban <cristiano.urban@inaf.it>
parent 70a6c156

transfer_service/Dockerfile

+5 −2
Original line number Diff line number Diff line
@@ -28,12 +28,15 @@ RUN mkdir /home/transfer_service/store && touch /home/transfer_service/store/foo 
    chown -R curban:curban /home/curban/store && \

    chmod -R 755 /home/curban/store && \

    mkdir /home/ccarbone/store && touch /home/ccarbone/store/foo3.txt && \

    echo "foo3" > /home/ccarbone/store/foo3.txt && \

    chown -R ccarbone:ccarbone /home/ccarbone/store && \

    chmod -R 755 /home/ccarbone/store && \

    mkdir /home/sbertocco/store && touch /home/sbertocco/store/foo4.txt && \

    echo "foo4" > /home/sbertocco/store/foo4.txt && \

    chown -R sbertocco:sbertocco /home/sbertocco/store && \

    chmod -R 755 /home/sbertocco/store && \

    mkdir /home/szorba/store && touch /home/szorba/store/foo5.txt && \

    mkdir -p /home/szorba/store/aaa/bbb && touch /home/szorba/store/aaa/bbb/foo5.txt && \

    echo "foo5" > /home/szorba/store/aaa/bbb/foo5.txt && \

    chown -R szorba:szorba /home/szorba/store && \

    chmod -R 755 /home/szorba/store

transfer_service/db_connector.py

+5 −4
Original line number Diff line number Diff line
@@ -66,12 +66,13 @@ class DbConnector(object): 
            out.close()

            #print(f"parentLtreePath: {parentLtreePath}, type: {type(parentLtreePath)}")

            self.cursor.execute("""

                INSERT INTO Node(parent_path, parent_relative_path, name, type, owner_id, creator_id, content_md5) 

                VALUES (%s, %s, %s, %s, %s, %s, %s);

                INSERT INTO Node(parent_path, parent_relative_path, name, tstamp_wrapper_dir, type, owner_id, creator_id, content_md5) 

                VALUES (%s, %s, %s, %s, %s, %s, %s, %s);

                """, 

                (parentLtreePath, 

                 parentLtreePath, 

                 node.name,

                 node.wrapperDir,

                 node.type, 

                 node.ownerID, 

                 node.creatorID,

transfer_service/node.py

+4 −0
Original line number Diff line number Diff line
@@ -8,6 +8,7 @@ class Node(object): 
    def __init__(self, name, type):

        self.parentPath = None

        self.name = name

        self.wrapperDir = None

        self.type = type

        self.format = None

        self.transferType = None
@@ -47,6 +48,9 @@ class Node(object): 
    def setName(self, name):

        self.name = name

        

    def setWrapperDir(self, wrapperDir):

        self.wrapperDir = wrapperDir



    def setType(self, type):

        if type in self.types:

            self.type = type

transfer_service/store_preprocessor.py

+30 −7
Original line number Diff line number Diff line
@@ -5,6 +5,7 @@ 


import json

import os

import re

import shutil

import sys
@@ -140,17 +141,30 @@ class StorePreprocessor(Preprocessor): 
        out.write(f"USER_ID: {self.userId}\n")

        #pathPrefix = self.storageBasePath.replace("{username}", self.username)

        pathPrefix = self.storageStorePath.replace("{username}", self.username)

        tstampWrapperDirPattern = re.compile("/[0-9]{4}_[0-9]{2}_[0-9]{2}-[0-9]{2}_[0-9]{2}_[0-9]{2}")

        for dir in dirs:

            out.write(f"DIR: {dir}\n")            

            out.write(f"pathPrefix: {pathPrefix}\n")

            basePath = os.path.dirname(dir).replace(pathPrefix, "/" + self.username)

            out.write(f"basePath: {basePath}\n")

            cnode = Node(os.path.basename(dir), "container");            

            nodeName = os.path.basename(dir)

            out.write(f"nodeName: {nodeName}\n")

            cnode = Node(nodeName, "container")

            if not tstampWrapperDirPattern.match("/" + nodeName):

                if tstampWrapperDirPattern.search(basePath):

                    tstampWrapperDir = tstampWrapperDirPattern.search(basePath).group(0).lstrip('/')

                    out.write(f"tstampWrapperDir: {tstampWrapperDir}\n")

                    #basePath = tstampWrapperDirPattern.sub("", basePath).rstrip('/')

                    basePath = tstampWrapperDirPattern.sub("", basePath)

                    out.write(f"newBasePath: {basePath}\n")

                    cnode.setWrapperDir(tstampWrapperDir)

                cnode.setParentPath(basePath)

                cnode.setOwnerID(self.userId)

                cnode.setCreatorID(self.userId)

                self.dbConn.insertNode(cnode, cnode.parentPath)

                

        out.write("\n\n")

         

        for flist in files:

            for file in flist:

                out.write(f"FILE: {file}\n")
@@ -158,7 +172,16 @@ class StorePreprocessor(Preprocessor): 
                    out.write(f"pathPrefix: {pathPrefix}\n")

                    basePath = os.path.dirname(file).replace(pathPrefix, "/" + self.username)

                    out.write(f"basePath: {basePath}\n")

                    dnode = Node(os.path.basename(file), "data")

                    nodeName = os.path.basename(file)

                    out.write(f"nodeName: {nodeName}\n")

                    dnode = Node(nodeName, "data")

                    #if not tstampWrapperDirPattern.match(nodeName):

                    if tstampWrapperDirPattern.search(basePath):

                        tstampWrapperDir = tstampWrapperDirPattern.search(basePath).group(0).lstrip('/')

                        out.write(f"tstampWrapperDir: {tstampWrapperDir}\n")

                        basePath = tstampWrapperDirPattern.sub("", basePath)

                        out.write(f"newBasePath: {basePath}\n")

                        dnode.setWrapperDir(tstampWrapperDir)

                    dnode.setParentPath(basePath)

                    dnode.setOwnerID(self.userId)

                    dnode.setCreatorID(self.userId)