Commit 942cfe32 authored by Jay's avatar Jay
Browse files

Adds just_stage option to apply

parent 02a46188

autocnet/graph/network.py

+14 −0
Original line number Original line Diff line number Diff line
@@ -1699,6 +1699,7 @@ class NetworkCandidateGraph(CandidateGraph): 
            queue=None,

            queue=None,

            redis_queue='processing_queue',

            redis_queue='processing_queue',

            exclude=None,

            exclude=None,

            just_stage=False,

            **kwargs):

            **kwargs):

        """

        """

        A mirror of the apply function from the standard CandidateGraph object. This implementation

        A mirror of the apply function from the standard CandidateGraph object. This implementation
@@ -1776,6 +1777,13 @@ class NetworkCandidateGraph(CandidateGraph): 
                      The redis queue to push messages to that are then pulled by the

                      The redis queue to push messages to that are then pulled by the

                      cluster job this call launches. Options are: 'processing_queue' (default)

                      cluster job this call launches. Options are: 'processing_queue' (default)

                      or 'working_queue'

                      or 'working_queue'



        just_stage : bool

                     If True, push messages to the redis queue for processing, but do not

                     submit a slurm/sbatch job to the cluster. This is useful when one process

                     is being used to orchestrate queue population and another process is being

                     used to process the messages. Default: False.



        Returns

        Returns

        -------

        -------

        job_str : str

        job_str : str
@@ -1854,6 +1862,10 @@ class NetworkCandidateGraph(CandidateGraph): 
            job += ' --queue'  # Use queue mode where jobs run until the queue is empty

            job += ' --queue'  # Use queue mode where jobs run until the queue is empty

        command = f'{condasetup} && {isissetup} && srun {job}'

        command = f'{condasetup} && {isissetup} && srun {job}'





        # The user does not want to submit the job. Only stage the messages.

        if just_stage:

            return command



        if queue == None:

        if queue == None:

            queue = self.config['cluster']['queue']

            queue = self.config['cluster']['queue']
@@ -1864,6 +1876,8 @@ class NetworkCandidateGraph(CandidateGraph): 
                     partition=queue,

                     partition=queue,

                     ntasks=ntasks,

                     ntasks=ntasks,

                     output=log_dir+f'/autocnet.{function}-%j')

                     output=log_dir+f'/autocnet.{function}-%j')



        # Submit the jobs to the cluster   

        if ntasks > 1:

        if ntasks > 1:

            job_str = submitter.submit(exclude=exclude)

            job_str = submitter.submit(exclude=exclude)

        else:

        else: