Commit fa18ed63 authored by Cristiano Urban's avatar Cristiano Urban
Browse files

Added 'results' dir for node import operation + added check on email... 


Added 'results' dir for node import operation + added check on email attachments for a nuber of nodes greater than a certain threshold.

Signed-off-by: default avatarCristiano Urban <cristiano.urban@inaf.it>
parent 12bcff38

transfer_service/Dockerfile

+2 −2
Original line number Diff line number Diff line
@@ -23,8 +23,8 @@ RUN chmod -R 755 /etc/vos_ts 
# Configure crontab

RUN /usr/bin/crontab /etc/vos_ts/cron_cleaner



# Create directory for logs

RUN mkdir -p /var/log/vos_ts

# Create directory for logs and results

RUN mkdir -p /var/log/vos_ts/results



# Copy files for import test

COPY test_import/testdir/ /home/test_import/testdir/

transfer_service/config/vos_ts.conf

+2 −0
Original line number Diff line number Diff line
@@ -89,6 +89,8 @@ admin_email = cristiano.urban@inaf.it 
log_level = DEBUG

; physical path on disk where log files are stored

log_dir = /var/log/vos_ts

; dir to store results of storage and import operations

res_dir = ${log_dir}/results





####################

transfer_service/import_executor.py

+13 −7
Original line number Diff line number Diff line
@@ -48,6 +48,7 @@ class ImportExecutor(TaskExecutor): 
        logStreamHandler.setFormatter(logFormatter)

        self.logger.addHandler(logFileHandler)

        self.logger.addHandler(logStreamHandler)

        self.resDir = params["res_dir"]

        self.systemUtils = SystemUtils()

        self.jobObj = None

        self.jobId = None
@@ -67,7 +68,7 @@ class ImportExecutor(TaskExecutor): 
        start = dt.now()

        nodeList = []

        timestamp = dt.now().strftime("%Y_%m_%d-%H_%M_%S")

        nodeListFile = "vos_import_report-" + timestamp

        nodeListFile = self.resDir + '/' + "vos_import_report-" + timestamp

        nlfp = open(nodeListFile, "w")



        #out = open("import_amqp_server_log.txt", "a")
@@ -187,14 +188,20 @@ class ImportExecutor(TaskExecutor): 
        Skipped nodes: {sum(res[-1] == 'SKIPPED' for res in nodeList)}



        """

        m.setMessageWithAttachment("VOSpace import notification", msg, nodeListFile)

        m.send()

        

        os.remove(nodeListFile)

        if len(nodeList) <= 4 * (10 ** 3):            

            m.setMessageWithAttachment("VOSpace import notification", msg, nodeListFile)

        else:

            info = f"""

            INFO: this operation involved a number of nodes greater than 10^5,

                  you will find the results in {self.resDir}.



            """

            msg += info

            m.setMessage("VOSpace import notification", msg)

        m.send()



    def run(self):

        #print("Starting import executor...")

        self.logger.info("Starting import executor...")

        self.setSourceQueueName("import_ready")

        self.setDestinationQueueName("import_terminated")
@@ -214,4 +221,3 @@ class ImportExecutor(TaskExecutor): 
                self.destQueue.insertJob(self.jobObj)

                self.srcQueue.extractJob()

                self.logger.info(f"Job {self.jobObj.jobId} MOVED from {self.srcQueue.name()} to {self.destQueue.name()}")
 
                #print(f"Job {self.jobObj.jobId} MOVED from {self.srcQueue.name()} to {self.destQueue.name()}")