Commit 0ac05ef9 authored by Cristiano Urban's avatar Cristiano Urban
Browse files

Renamed 'store' and 'retrieve' directories.

parent 6d5c7eb3
Loading
Loading
Loading
Loading
+3 −3
Original line number Diff line number Diff line
@@ -78,10 +78,10 @@ You can access the *transfer_service* container with:
docker exec -it transfer_service bash
```
On this container, hosted on the so-called transfer node, each user will find his/her own home folder and two subfolders representing respectively the entry point and exit point for the user data:
- */home/name.surname/store*
- */home/name.surname/retrieve*
- */home/name.surname/vospace_store*
- */home/name.surname/vospace_retrieve*

The user will copy the data to be stored within the *store* folder, while he/she will find the requested data within the *retrieve* folder.
The user will copy the data to be stored within the *vospace_store* folder, while he/she will find the requested data within the *vospace_retrieve* folder.
This use case was implemented in order to try to offer support to users providing huge amounts of data in the order of terabytes.

Log files can be found in */var/log/vos_ts*.
+25 −25
Original line number Diff line number Diff line
@@ -37,31 +37,31 @@ COPY test_import/testdir/ /home/test_import/testdir/
COPY test_import/testdir-md5sum.txt /home/test_import/
RUN chmod -R 755 /home/test_import

# Create a 'store' directory with some files for various users
RUN mkdir /home/cristiano.urban/store /home/cristiano.urban/retrieve && \
    echo "foo?!" > /home/cristiano.urban/store/foo2.txt && \
    mkdir /home/cristiano.urban/store/mydir && \
    echo "Another foo!" > /home/cristiano.urban/store/mydir/another_foo2.txt && \
    mkdir /home/cristiano.urban/store/mydir/dir2 && \
    echo "fooX" > /home/cristiano.urban/store/mydir/dir2/fooX.txt && \
    chown -R cristiano.urban:cristiano.urban /home/cristiano.urban/store && \
    chown -R cristiano.urban:cristiano.urban /home/cristiano.urban/retrieve && \
    chmod -R 755 /home/cristiano.urban/store && \ 
    chmod -R 755 /home/cristiano.urban/retrieve && \
    mkdir /home/sara.bertocco/store /home/sara.bertocco/retrieve && \
    touch /home/sara.bertocco/store/foo4.txt && \
    echo "foo4" > /home/sara.bertocco/store/foo4.txt && \
    chown -R sara.bertocco:sara.bertocco /home/sara.bertocco/store && \
    chown -R sara.bertocco:sara.bertocco /home/sara.bertocco/retrieve && \
    chmod -R 755 /home/sara.bertocco/store && \
    chmod -R 755 /home/sara.bertocco/retrieve && \
    mkdir -p /home/sonia.zorba/store/aaa/bbb && mkdir /home/sonia.zorba/retrieve && \
    touch /home/sonia.zorba/store/aaa/bbb/foo5.txt && \
    echo "foo5" > /home/sonia.zorba/store/aaa/bbb/foo5.txt && \
    chown -R sonia.zorba:sonia.zorba /home/sonia.zorba/store && \
    chown -R sonia.zorba:sonia.zorba /home/sonia.zorba/retrieve && \    
    chmod -R 755 /home/sonia.zorba/store && \
    chmod -R 755 /home/sonia.zorba/retrieve
# Create a 'vospace_store' directory with some files for various users
RUN mkdir /home/cristiano.urban/vospace_store /home/cristiano.urban/vospace_retrieve && \
    echo "foo?!" > /home/cristiano.urban/vospace_store/foo2.txt && \
    mkdir /home/cristiano.urban/vospace_store/mydir && \
    echo "Another foo!" > /home/cristiano.urban/vospace_store/mydir/another_foo2.txt && \
    mkdir /home/cristiano.urban/vospace_store/mydir/dir2 && \
    echo "fooX" > /home/cristiano.urban/vospace_store/mydir/dir2/fooX.txt && \
    chown -R cristiano.urban:cristiano.urban /home/cristiano.urban/vospace_store && \
    chown -R cristiano.urban:cristiano.urban /home/cristiano.urban/vospace_retrieve && \
    chmod -R 755 /home/cristiano.urban/vospace_store && \
    chmod -R 755 /home/cristiano.urban/vospace_retrieve && \
    mkdir /home/sara.bertocco/vospace_store /home/sara.bertocco/vospace_retrieve && \
    touch /home/sara.bertocco/vospace_store/foo4.txt && \
    echo "foo4" > /home/sara.bertocco/vospace_store/foo4.txt && \
    chown -R sara.bertocco:sara.bertocco /home/sara.bertocco/vospace_store && \
    chown -R sara.bertocco:sara.bertocco /home/sara.bertocco/vospace_retrieve && \
    chmod -R 755 /home/sara.bertocco/vospace_store && \
    chmod -R 755 /home/sara.bertocco/vospace_retrieve && \
    mkdir -p /home/sonia.zorba/store/aaa/bbb && mkdir /home/sonia.zorba/vospace_retrieve && \
    touch /home/sonia.zorba/vospace_store/aaa/bbb/foo5.txt && \
    echo "foo5" > /home/sonia.zorba/vospace_store/aaa/bbb/foo5.txt && \
    chown -R sonia.zorba:sonia.zorba /home/sonia.zorba/vospace_store && \
    chown -R sonia.zorba:sonia.zorba /home/sonia.zorba/vospace_retrieve && \
    chmod -R 755 /home/sonia.zorba/vospace_store && \
    chmod -R 755 /home/sonia.zorba/vospace_retrieve

# Run a shell
CMD /bin/bash
+2 −2
Original line number Diff line number Diff line
@@ -135,6 +135,6 @@ res_dir = ${log_dir}/results
; user folder
base_path = /home/{username}
; data entry point
store_path = ${base_path}/store
store_path = ${base_path}/vospace_store
; data exit point
retrieve_path = ${base_path}/retrieve
retrieve_path = ${base_path}/vospace_retrieve
+4 −4
Original line number Diff line number Diff line
@@ -134,7 +134,7 @@ class DataRPCServer(RedisRPCServer):
                             "errorCode": 2,
                             "errorMsg": errorMsg }
                return response
            #folderPath = "/home/" + username + "/store"
            #folderPath = "/home/" + username + "/vospace_store"
            folderPath = self.storageStorePath.replace("{username}", username)
            userInfo = self.systemUtils.userInfo(username)
            # Check if the user exists on the transfer node and is registered in the database
@@ -173,7 +173,7 @@ class DataRPCServer(RedisRPCServer):
                gid = userInfo[2]
                # Avoid privilege escalation
                if uid >= 1000 and gid >= 100:
                    # If write permissions are set and the 'store' folder is not empty,
                    # If write permissions are set and the 'vospace_store' folder is not empty,
                    # it means that data is ready to be copied, otherwise, nothing can
                    # be done until the write permissions are restored or new data is
                    # copied on the transfer node by the user.
@@ -181,7 +181,7 @@ class DataRPCServer(RedisRPCServer):
                        response = { "responseType": "STORE_ACK",
                                     "storageList": storageList }
                    elif os.access(folderPath, os.W_OK) and not os.listdir(folderPath):
                        errorMsg = "The 'store' directory on the transfer node is empty."
                        errorMsg = f"The '{folderPath}' directory on the transfer node is empty."
                        self.logger.error(errorMsg)
                        response = { "responseType": "ERROR",
                                     "errorCode": 12,
@@ -257,7 +257,7 @@ class DataRPCServer(RedisRPCServer):
        return response

    def prepare(self, username):
        #path = "/home/" + username + "/store"
        #path = "/home/" + username + "/vospace_store"
        path = self.storageStorePath.replace("{username}", username)
        for el in os.listdir(path):
            try:
+1 −1
Original line number Diff line number Diff line
@@ -75,7 +75,7 @@ class RetrieveCleaner(TaskExecutor):
        #   loop over the two lists (nodeList and destPathList):
        #     if the vospace node is not busy: 
        #       set 'async_trans' = True
        #       delete the file/dir in the 'retrieve' directory
        #       delete the file/dir in the 'vospace_retrieve' directory
        #       remove the corresponding elements on the two lists
        try:
            self.logger.info("++++++++++ Start of execution phase ++++++++++")
Loading