Commit a22ad5b9 authored by Kelvin Rodriguez's avatar Kelvin Rodriguez Committed by Jesse Mapel
Browse files

adding some missing targets into rclone.conf (#5042) 

* adding some missing targets

* added kwargs to download scripts

* removed unused code

* address comments
parent d4e13ec4

isis/config/rclone.conf

+16 −0
Original line number Diff line number Diff line
@@ -16,6 +16,10 @@ url = http://naif.jpl.nasa.gov/ 
type = http

url = http://www.darts.isas.jaxa.jp/



[control_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/control/



[apollo15_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/apollo15
@@ -64,6 +68,14 @@ remote = jaxa:/pub/hayabusa2/spice_bundle/spice_kernels 
type = alias

remote = asc_s3:asc-isisdata/usgs_data/hayabusa2



[gallileo_naifKernels]

type = alias 

remote = naif:/pub/naif/GLL/kernels



[gallileo_usgs]

type = alias 

remote = asc_s3:asc-isisdata/usgs_data/galileo



[juno_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/jno-j_e_ss-spice-6-v1.0/jnosp_1000/data/
@@ -140,6 +152,10 @@ remote = naif:/pub/naif/M10/kernels 
type = alias

remote = asc_s3:asc-isisdata/usgs_data/mariner10



[mer_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/mer_usgs



[messenger_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/mess-e_v_h-spice-6-v1.0/messsp_1000/data/

isis/scripts/downloadIsisData

+40 −27
Original line number Diff line number Diff line
@@ -88,7 +88,7 @@ def rclone(command, config=None, extra_args=[], redirect_stdout=True, redirect_s 
        raise Exception(message)





def create_rclone_arguments(destination, mission_name, dry_run=False, ntransfers=10):

def create_rclone_arguments(destination, mission_name, ntransfers=10, rclone_kwargs=[]):

    """

    Parameters

    ----------
@@ -110,20 +110,14 @@ def create_rclone_arguments(destination, mission_name, dry_run=False, ntransfers 
    if source_type == "naifKernels":

        destination = os.path.join(destination, "kernels")



    if not os.path.exists(destination) and dry_run==False:

        log.debug("{destination} does not exist, making directory")

        os.makedirs(destination)



    extra_args=[f"{mission_name}",f"{destination}", "--progress", f"--checkers={ntransfers}", f"--transfers={ntransfers}", "--track-renames", f"--log-level={log.getLevelName(log.getLogger().getEffectiveLevel())}"]



    if dry_run:

        extra_args.append("--dry-run")



    extra_args.extend(rclone_kwargs)

    log.debug(f"Args created: {extra_args}")

    return extra_args





def main(mission, dest, cfg_path, dry_run, ntransfers):

def main(mission, dest, cfg_path, ntransfers, kwargs):

    """

    Parameters

    ----------
@@ -133,6 +127,10 @@ def main(mission, dest, cfg_path, dry_run, ntransfers): 


    dest : str

               path to location where files will be copied/downloaded too

    

    kwargs : list

             list of additional args to pass into rclone

          

    """

    log.info("---Script starting---")
@@ -159,35 +157,50 @@ def main(mission, dest, cfg_path, dry_run, ntransfers): 
        raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions.keys()}")



    if mission == "legacybase":

        args = create_rclone_arguments(dest, "legacybase_usgs:", dry_run, ntransfers)

        rclone(command="sync", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

        args = create_rclone_arguments(dest, "legacybase_usgs:", ntransfers, kwargs)

        rclone(command="copy", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

    elif(mission.upper() == "ALL"):

        supported_missions.pop("legacybase")

        for mission, remotes in supported_missions.items():

            for remote in remotes:

                args = create_rclone_arguments(dest, remote, dry_run, ntransfers)

                rclone(command="sync", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

                args = create_rclone_arguments(dest, remote, ntransfers, kwargs)

                rclone(command="copy", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

    else:

        for remote in supported_missions[mission]:

            args = create_rclone_arguments(dest, remote, dry_run, ntransfers)

            rclone(command="sync", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

            args = create_rclone_arguments(dest, remote,  ntransfers, kwargs)

            rclone(command="copy", extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)





if __name__ == '__main__':

    usageString = "usage: downloadIsisData mission dest [-h] [-v] [-n NUM_TRANSFERS] [--config CONFIG] [rclone flags]"

    helpString = (

    '''This will allow for a user to download isis data directly to their machine from the USGS S3 buckets as well as public end points\n\n'''

    '''<Mission name> <destination to copy to> <--dry-run (optional)> \n\n'''

    '''Example of how to run this program:\n\n'''

    '''    downloadIsisData tgo ~/isisData/\n\n'''

    '''NOTE: if you would like to download the data for every mission supply the value ALL for the <Mission name> parameter''')

    parser = argparse.ArgumentParser(description = helpString, formatter_class=argparse.RawTextHelpFormatter)

    parser.add_argument('mission', help='mission for files to be downloaded')

    '''

    Use to copy ISIS data from USGS sources as well as public end points. 



    This script combines multiple sources using the rclone command under the hood. 

    

    After positional args, accepts arbitrary rclone flags. Run "rclone help flags" to get a full list of rclone flags. 

    

    See the ISIS repo (https://github.com/USGS-Astrogeology/ISIS3) for more info. 

    

    Examples of how to run this program:

        

        # download everything for tgo

        $ downloadIsisData tgo $ISISDATA

    

        # download everything except kernels with debug output

        $ downloadIsisData all $ISISDATA --dry-run --exclude="kernels/**" -vv

        

        # download only cks and fks

        $ downloadIsisData all $ISISDATA --include="{ck/**,fk/**}" 

    ''')

    parser = argparse.ArgumentParser(description = helpString,formatter_class=argparse.RawDescriptionHelpFormatter, usage=usageString)

    parser.add_argument('mission', help='mission for files to be downloaded. Use "all" to download everything, "legacybase" to download base data for ISIS<4.1.0.')

    parser.add_argument('dest', help='the destination to download files from source')

    parser.add_argument('--dry-run', help="run a dry run for rclone value should be a boolean", default=False, action='store_true')

    parser.add_argument('-v', '--verbose', action='count', default=0)

    parser.add_argument('-n', '--num-transfers', action='store', default=10)

    parser.add_argument('--config', action='store', default=find_conf())

    args = parser.parse_args()

    args, kwargs = parser.parse_known_args()

    

    log_kwargs = {

      'format': '%(asctime)s %(levelname)-8s %(message)s',
@@ -203,7 +216,7 @@ if __name__ == '__main__': 
        log_kwargs['level'] = log.DEBUG



    log.basicConfig(**log_kwargs)

    log.debug("Additional Args:", kwargs)



    main(args.mission, args.dest, os.path.expanduser(args.config), args.dry_run, args.num_transfers)



    main(args.mission, args.dest, os.path.expanduser(args.config), args.num_transfers, kwargs)