Unverified Commit 70ec72a4 authored by Amy Stamile's avatar Amy Stamile Committed by GitHub
Browse files

fixes rclone recopying existing kernels (#5255) 

* fixes rclone duplication

* updated changelog

* Updated script

* Addressed PR feedback
parent 4c33e0a8

CHANGELOG.md

+1 −0
Original line number Diff line number Diff line
@@ -57,6 +57,7 @@ release. 
### Fixed

- Updated History constructor to check for invalid BLOB before copying History BLOB to output cube [#4966](https://github.com/DOI-USGS/ISIS3/issues/4966)

- Updated photomet MinnaertEmpirical model to support photemplate-style PVL format [#3621](https://github.com/DOI-USGS/ISIS3/issues/3621)

- Fixed downloadIsisData script copying existing files with every rclone download [#5245](https://github.com/DOI-USGS/ISIS3/issues/5245)

- Fixed gaussstretch segmentation fault error and refactored gaussstretch files/tests to use gtest [#5240](https://github.com/DOI-USGS/ISIS3/issues/5240)

- Fix matrix inversion errors in <i>findfeatures</i> due to bad FASTGEOM matrix transforms using a more robust implementation to detect these errors and throw exceptions. Images with these errors are captured and logged to the <b>TONOTMATCHED</b> file. Fixes [#4639](https://github.com/DOI-USGS/ISIS3/issues/4639)

- Fixed <i>findfeatures</i> use of projected mosaics with correct check for <b>TargetName</b> in the Mapping labels. [#4772](https://github.com/DOI-USGS/ISIS3/issues/4772)

isis/config/rclone.conf

+176 −70
Original line number Diff line number Diff line
@@ -20,256 +20,362 @@ url = http://www.darts.isas.jaxa.jp/ 
type =http

url = https://sbnarchive.psi.edu/pds3/



[control_usgs]

[base]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/base/



[legacybase]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/legacy_base/



[control]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/control/



[apollo15_usgs]

[apollo15]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/apollo15/



[apollo16_usgs]

[apollo16]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/apollo16/



[apollo17_usgs]

[apollo17]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/apollo17/



[lro_usgs]

[lro]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/lro/



[tgo_naifKernels]

type = alias

remote = esa:/data/SPICE/ExoMars2016/kernels/

type = combine

upstreams = kernels=esa:/data/SPICE/ExoMars2016/kernels/



[tgo_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/tgo/



[tgo]

type = union 

upstreams = tgo_usgs: tgo_naifKernels:



[dawn_naifKernels]

type = alias

remote = naif:/pub/naif/DAWN/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/DAWN/kernels/



[dawn_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/dawn/



[dawn_pck]

type =alias

remote = sbn:dawn/fc/DWNCHSPG_2/GEOMETRY/

type = combine

upstreams = pck=sbn:dawn/fc/DWNCHSPG_2/GEOMETRY/



[dawn_kernel_combine]

type = combine

upstreams = kernels=dawn_pck:



[dawn]

type = union

upstreams = dawn_kernel_combine: dawn_naifKernels: dawn_usgs:



[cassini_naifKernels]

type = alias

remote = naif:/pub/naif/CASSINI/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/CASSINI/kernels/



[cassini_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/cassini/



[cassini]

type = union 

upstreams = cassini_usgs: cassini_naifKernels:



[hayabusa2_naifKernels]

type = alias

remote = jaxa:/pub/hayabusa2/spice_bundle/spice_kernels/

type = combine

upstreams = kernels=jaxa:/pub/hayabusa2/spice_bundle/spice_kernels/



[hayabusa2_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/hayabusa2/



[hayabusa2]

type = union 

upstreams = hayabusa2_usgs: hayabusa2_naifKernels:



[galileo_naifKernels]

type = alias 

remote = naif:/pub/naif/GLL/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/GLL/kernels/



[galileo_usgs]

type = alias 

remote = asc_s3:asc-isisdata/usgs_data/galileo/



[galileo]

type = union

upstreams = galileo_usgs: galileo_naifKernels:



[juno_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/jno-j_e_ss-spice-6-v1.0/jnosp_1000/data/

type = combine

upstreams = kernels=naif:/pub/naif/JUNO/kernels/



[juno_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/juno/



[lo_usgs]

[juno]

type = union 

upstreams = juno_usgs: juno_naifKernels:



[lo]

type = alias 

remote = asc_s3:asc-isisdata/usgs_data/lo/



[odyssey_naifKernels]

type = alias

remote = naif:/pub/naif/M01/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/M01/kernels/



[odyssey_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/odyssey/



[odyssey]

type = union 

upstreams = odyssey_usgs: odyssey_naifKernels: 



[mro_naifKernels]

type = alias

remote = naif:/pub/naif/MRO/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/MRO/kernels/



[mro_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/mro/



[mro]

type = union

upstreams = mro_usgs: mro_naifKernels:



[mex_naifKernels]

type = alias

remote = naif:/pub/naif/MEX/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/MEX/kernels/



[mex_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/mex/



[legacybase_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/legacy_base/

[mex]

type = union 

upstreams = mex_usgs: mex_naifKernels:



[hayabusa_naifKernels]

type = alias

remote = jaxa:/pub/spice/HAYABUSA/kernels/

type = combine

upstreams = kernels=jaxa:/pub/spice/HAYABUSA/kernels/



[hayabusa_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/hayabusa/



[hayabusa]

type = union 

upstreams = hayabusa_usgs: hayabusa_naifKernels:



[chandrayaan1_naifKernels]

type = alias

remote = esa:/data/SPICE/CHANDRAYAAN-1/kernels/

type = combine

upstreams = kernels=esa:/data/SPICE/CHANDRAYAAN-1/kernels/



[chandrayaan1_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/chandrayaan1/



[chandrayaan1]

type = union 

upstreams = chandrayaan1_usgs: chandrayaan1_naifKernels:



[clementine1_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/clem1-l-spice-6-v1.0/clsp_1000/data/

type = combine

upstreams = naif:/pub/naif/CLEMENTINE/kernels/



[clementine1_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/clementine1/



[clementine1]

type = union

upstreams = clementine1_usgs: clementine1_naifKernels:



[kaguya_naifKernels]

type = alias

remote = jaxa:/pub/pds3/sln-l-spice-6-v1.0/slnsp_1000/data/

type = combine

upstreams = kernels=jaxa:/pub/spice/SELENE/kernels/



[kaguya_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/kaguya/



[kaguya]

type = union 

upstreams = kaguya_usgs: kaguya_naifKernels:



[mariner10_naifKernels]

type = alias

remote = naif:/pub/naif/M10/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/M10/kernels/



[mariner10_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/mariner10/



[mer_usgs]

[mariner10]

type = union

upstreams = mariner10_usgs: mariner10_naifKernels:



[mer]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/mer/



[messenger_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/mess-e_v_h-spice-6-v1.0/messsp_1000/data/

type = combine

upstreams = kernels=naif:/pub/naif/pds/data/mess-e_v_h-spice-6-v1.0/messsp_1000/data/kernels/



[messenger_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/messenger/



[messenger]

type = union 

upstreams = messenger_usgs: messenger_naifKernels:



[mgs_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/mgs-m-spice-6-v1.0/mgsp_1000/data/

type = combine

upstreams = kernels=naif:/pub/naif/pds/data/mgs-m-spice-6-v1.0/mgsp_1000/data/kernels/



[mgs_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/mgs/



[mgs]

type = union 

upstreams = mgs_usgs: mgs_naifKernels:



[near_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/near-a-spice-6-v1.0/nearsp_1000/data/

type = combine

upstreams = kernels=naif:/pub/naif/pds/data/near-a-spice-6-v1.0/nearsp_1000/data/



[near_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/near/



[near]

type = union 

upstreams = near_usgs: near_naifKernels:



[newhorizons_naifKernels]

type = alias

remote = naif:/pub/naif/pds/data/nh-j_p_ss-spice-6-v1.0/nhsp_1000/data/

type = combine

upstreams = kernels=naif:/pub/naif/NEWHORIZONS/kernels/



[newhorizons_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/newhorizons/



[newhorizons]

type = union 

upstreams = newhorizons_usgs: newhorizons_naifKernels:



[osirisrex_naifKernels]

type = alias

remote = naif:/pub/naif/pds/pds4/orex/orex_spice/spice_kernels/

type = combine

upstreams = kernels=naif:/pub/naif/pds/pds4/orex/orex_spice/spice_kernels/ 



[osirisrex_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/osirisrex/



[rolo_usgs]

[osirisrex]

type = union

upstreams = osirisrex_usgs: osirisrex_naifKernels:



[rolo]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/rolo/



[rosetta_naifKernels]

type = alias

remote = naif:/pub/naif/ROSETTA/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/ROSETTA/kernels/



[rosetta_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/rosetta/



[rosetta]

type = union 

upstreams = rosetta_usgs: rosetta_naifKernels:



[smart1_naifKernels]

type = alias

remote = esa:/data/SPICE/SMART-1/kernels/

type = combine

upstreams = kernels=esa:/data/SPICE/SMART-1/kernels/



[smart1_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/smart1/



[smart1]

type = union 

upstreams = smart1_usgs: smart1_naifKernels:



[viking1_naifKernels]

type = alias

remote = naif:/pub/naif/VIKING/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/VIKING/kernels/



[viking1_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/viking1/



[viking1]

type = union 

upstreams = viking1_usgs: viking1_naifKernels: 



[viking2_naifKernels]

type = alias

remote = naif:/pub/naif/VIKING/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/VIKING/kernels/



[viking2_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/viking2/



[viking2]

type = union

upstreams = viking2_usgs: viking2_naifKernels:



[voyager1_naifKernels]

type = alias

remote = naif:/pub/naif/VOYAGER/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/VOYAGER/kernels/



[voyager1_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/voyager1/



[voyager1]

type = union 

upstreams = voyager1_usgs: voyager1_naifKernels:



[voyager2_naifKernels]

type = alias

remote = naif:/pub/naif/VOYAGER/kernels/

type = combine

upstreams = kernels=naif:/pub/naif/VOYAGER/kernels/



[voyager2_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/voyager2/



[base_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/base/

[voyager2]

type = union 

upstreams = voyager2_usgs: voyager2_naifKernels:



[msl_usgs]

type = alias

remote = asc_s3:asc-isisdata/usgs_data/msl/



[msl_naifKernels]

type = alias

remote = naif:/pub/naif/MSL/kernels/



type = combine

upstreams = kernels=naif:/pub/naif/MSL/kernels/



[msl]

type = union 

upstreams = msl_usgs: msl_naifKernels:
 
 No newline at end of file

isis/scripts/downloadIsisData

+20 −32
Original line number Diff line number Diff line
@@ -123,17 +123,14 @@ def create_rclone_arguments(destination, mission_name, parsedArgs, rclone_kwargs 
        set of kernels that will be downloaded

    """

    log.debug(f"Creating RClone command for {mission_name}")

    mission_dir_name, source_type = mission_name.replace(":", "").split("_")



    if (mission_dir_name == "legacybase"):

    mission_dir_name = mission_name

    

    if (mission_name == "legacybase"):

        # We still want things to go into base

        mission_dir_name = "base"



    log.debug(f"Mission_dir_name: {mission_dir_name}, source_type: {source_type}")



    destination = os.path.join(destination, str(mission_dir_name).replace(":",""))

    if source_type == "naifKernels":

        destination = os.path.join(destination, "kernels")

    destination = os.path.join(destination, str(mission_dir_name))



    if args.filter: 

        filters = [f"- {arg}" for arg in args.filters]
@@ -153,7 +150,7 @@ def create_rclone_arguments(destination, mission_name, parsedArgs, rclone_kwargs 
        filter_list.append("- *")



    filter_args = [f'--filter={item}' for item in filter_list]

    extra_args = [f"{mission_name}",

    extra_args = [f"{mission_name}:",

                  f"{destination}", 

                  "--progress",

                  f"--checkers={parsedArgs.num_transfers}",
@@ -189,35 +186,26 @@ def main(mission, dest, cfg_path, parsedArgs, kwargs): 
        quit(-1)

    log.debug(f"Remote Sources: {config_sources}")



    supported_missions = OrderedDict({})

    for source in sorted(config_sources, key=lambda x: x.split("_")[-1]):

        parsed_name = source.split("_")

        # If it is a mission, it should be in the format <mission_nam>_<source_type>

        if len(parsed_name) == 2 and parsed_name[1] in ["usgs:", "naifKernels:"]:

            remotes_mission_name = parsed_name[0]

            supported_missions[remotes_mission_name] = supported_missions.get(remotes_mission_name, []) + [source]

    supported_missions = []

    for source in config_sources:

        if not any(x in source for x in ["_", "esa", "naif", "jaxa", "sbn"]):

            if source != "":

                supported_missions.append(source.replace(":", ""))



    log.debug(f"Supported missions:\n {supported_missions.keys()}")

    log.debug(f"Complete Dictionary:\n {json.dumps(supported_missions, indent=2)}")

    log.debug(f"Supported missions:\n {supported_missions}")



    if not mission in supported_missions.keys() and mission.upper() not in ("ALL", "LEGACYBASE"):

        raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions.keys()}")

    if not mission in supported_missions and mission.upper() not in ("ALL", "LEGACYBASE"):

        raise LookupError(f"{mission} is not in the list of supported missions: {supported_missions}")



    if mission == "legacybase":

        args = create_rclone_arguments(dest, "legacybase_usgs:", parsedArgs, kwargs)

        rclone(command=parsedArgs.command, extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

    elif(mission.upper() == "ALL"):

        supported_missions.pop("legacybase")

        for mission, remotes in supported_missions.items():

            for remote in remotes:

                args = create_rclone_arguments(dest, remote, parsedArgs, kwargs)

    if(mission.upper() == "ALL"):

        supported_missions.remove("legacybase")

        for mission in supported_missions:

            args = create_rclone_arguments(dest, mission, parsedArgs, kwargs)

            rclone(command=parsedArgs.command, extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)

    else:

        for remote in supported_missions[mission]:

            args = create_rclone_arguments(dest, remote,  parsedArgs, kwargs)

        args = create_rclone_arguments(dest, mission,  parsedArgs, kwargs)

        rclone(command=parsedArgs.command, extra_args=args, config=cfg_path, redirect_stdout=False, redirect_stderr=False)





if __name__ == '__main__':

    usageString = "usage: downloadIsisData mission dest [-h] [-v] [-n NUM_TRANSFERS] [--config CONFIG] [rclone flags]"

    helpString = (