parameters: (data_path)
docstring: r"""Initializes data_sources and data_groups dicts with a variable data_path path.
docstring_summary: r"""Initializes data_sources and data_groups dicts with a variable data_path path.
docstring_tokens: r Initializes data_sources and data_groups dicts with a variable data_path path .
function: def initialize_test_data_sources(data_path):
      global data_sources
      global data_groups
      r"""Initializes data_sources and data_groups dicts with a variable data_path path.
      """
      # dict keyed by uids with download specs for various individual test assets and datasets
      data_sources = {
          # "example_uid": {
          #   "source": the URL download link
          #   "package_name": the filename of the downloaded compressed package
          #   "download_pre_args": (optional)(wget) commands preceding filename
          #   "download_post_args": (optional)(wget) commands follow filename
          #   "link": symlink to the data directory pointing to the active version directory
          #   "version": data version tag
          # }
          "habitat_test_scenes": {
              "source": "http://dl.fbaipublicfiles.com/habitat/habitat-test-scenes_v1.0.zip",
              "package_name": "habitat-test-scenes_v1.0.zip",
              "link": data_path + "scene_datasets/habitat-test-scenes",
              "version": "1.0",
          },
          "habitat_test_pointnav_dataset": {
              "source": "http://dl.fbaipublicfiles.com/habitat/habitat-test-pointnav-dataset_v1.0.zip",
              "package_name": "habitat-test-pointnav-dataset_v1.0.zip",
              "link": data_path + "datasets/pointnav/habitat-test-scenes",
              "version": "1.0",
          },
          "habitat_example_objects": {
              "source": "http://dl.fbaipublicfiles.com/habitat/objects_v0.2.zip",
              "package_name": "objects_v0.2.zip",
              "link": data_path + "objects/example_objects",
              "version": "0.2",
          },
          "locobot_merged": {
              "source": "http://dl.fbaipublicfiles.com/habitat/locobot_merged_v0.2.zip",
              "package_name": "locobot_merged_v0.2.zip",
              "link": data_path + "objects/locobot_merged",
              "version": "0.2",
          },
          "mp3d_example_scene": {
              "source": "http://dl.fbaipublicfiles.com/habitat/mp3d/mp3d_example_v1.1.zip",
              "package_name": "mp3d_example_v1.1.zip",
              "link": data_path + "scene_datasets/mp3d_example",
              "version": "1.1",
          },
          "coda_scene": {
              "source": "https://dl.fbaipublicfiles.com/habitat/coda_v1.0.zip",
              "package_name": "coda_v1.0.zip",
              "download_pre_args": "--no-check-certificate ",
              "download_post_args": " -O " + data_path + "coda_v1.0.zip",
              "link": data_path + "scene_datasets/coda",
              "version": "1.0",
          },
          "webxr_hand_demo": {
              "source": "https://dl.fbaipublicfiles.com/habitat/data/scene_datasets/webxr_hand_demo_data.zip",
              "package_name": "webxr_hand_demo_data.zip",
              "link": data_path + "webxr_hand_demo_dataset",
              "version": "1.0",
          },
          "replica_cad_dataset": {
              "source": "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/ReplicaCAD_dataset_v1.2.zip",
              "package_name": "ReplicaCAD_dataset_v1.2.zip",
              "link": data_path + "replica_cad",
              "version": "1.2",
          },
          "replica_cad_baked_lighting": {
              "source": "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/ReplicaCAD_baked_lighting_v1.1.zip",
              "package_name": "ReplicaCAD_baked_lighting_v1.1.zip",
              "link": data_path + "replica_cad_baked_lighting",
              "version": "1.1",
          },
          "ycb": {
              "source": "https://dl.fbaipublicfiles.com/habitat/ycb/hab_ycb_v1.1.zip",
              "package_name": "hab_ycb_v1.1.zip",
              "link": data_path + "objects/ycb",
              "version": "1.1",
          },
          "hab_fetch": {
              "source": "http://dl.fbaipublicfiles.com/habitat/hab_fetch_v1.0.zip",
              "package_name": "hab_fetch_v1.0.zip",
              "link": data_path + "robots/hab_fetch",
              "version": "1.0",
          },
          "rearrange_pick_dataset_v0": {
              "source": "https://dl.fbaipublicfiles.com/habitat/data/datasets/rearrange_pick/replica_cad/v0/rearrange_pick_replica_cad_v0.zip",
              "package_name": "rearrange_pick_replica_cad_v0.zip",
              "link": data_path + "datasets/rearrange_pick/replica_cad/v0",
              "version": "1.0",
          },
          "hab2_bench_assets": {
              "source": "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/hab2_bench_assets.zip",
              "package_name": "hab2_bench_assets.zip",
              "link": data_path + "hab2_bench_assets",
              "version": "1.0",
          },
      }
  
      data_sources.update(
          {
              f"hm3d_{split}_{data_format}": {
                  "source": "https://api.matterport.com/resources/habitat/hm3d-{split}-{data_format}.tar{ext}".format(
                      ext=".gz" if data_format == "obj+mtl" else "",
                      split=split,
                      data_format=data_format,
                  ),
                  "download_pre_args": "--location",
                  "package_name": "hm3d-{split}-{data_format}.tar{ext}".format(
                      ext=".gz" if data_format == "obj+mtl" else "",
                      split=split,
                      data_format=data_format,
                  ),
                  "link": data_path + "scene_datasets/hm3d",
                  "version": "1.0",
                  "version_dir": "hm3d-{version}/hm3d",
                  "extract_postfix": f"{split}",
                  "downloaded_file_list": f"hm3d-{{version}}/{split}-{data_format}-files.json.gz",
                  "requires_auth": True,
                  "use_curl": True,
                  "post_extract_fn": hm3d_train_configs_post
                  if split == "train" and data_format == "configs"
                  else None,
              }
              for split, data_format in itertools.product(
                  ["minival", "train", "val"],
                  ["glb", "obj+mtl", "habitat", "configs"],
              )
          }
      )
  
      data_sources.update(
          {
              f"hm3d_example_{data_format}": {
                  "source": "https://github.com/matterport/habitat-matterport-3dresearch/raw/main/example/hm3d-example-{data_format}.tar{ext}".format(
                      ext=".gz" if data_format == "obj+mtl" else "",
                      data_format=data_format,
                  ),
                  "package_name": "hm3d-example-{data_format}.tar{ext}".format(
                      ext=".gz" if data_format == "obj+mtl" else "",
                      data_format=data_format,
                  ),
                  "link": data_path + "scene_datasets/hm3d",
                  "version": "1.0",
                  "version_dir": "hm3d-{version}/hm3d",
                  "extract_postfix": "example",
                  "downloaded_file_list": f"hm3d-{{version}}/example-{data_format}-files.json.gz",
              }
              for data_format in ["glb", "obj+mtl", "habitat", "configs"]
          }
      )
  
      # data sources can be grouped for batch commands with a new uid
      data_groups = {
          "ci_test_assets": [
              "habitat_test_scenes",
              "habitat_test_pointnav_dataset",
              "habitat_example_objects",
              "locobot_merged",
              "mp3d_example_scene",
              "coda_scene",
              "replica_cad_dataset",
              "hab_fetch",
          ],
          "rearrange_task_assets": [
              "replica_cad_dataset",
              "hab_fetch",
              "ycb",
              "rearrange_pick_dataset_v0",
          ],
          "hm3d_example": ["hm3d_example_habitat", "hm3d_example_configs"],
          "hm3d_val": ["hm3d_val_habitat", "hm3d_val_configs"],
          "hm3d_train": ["hm3d_train_habitat", "hm3d_train_configs"],
          "hm3d_minival": ["hm3d_minival_habitat", "hm3d_minival_configs"],
          "hm3d_full": list(filter(lambda k: k.startswith("hm3d_"), data_sources.keys())),
      }
  
      data_groups.update(
          {
              f"hm3d_{split}_full": list(
                  filter(lambda k: k.startswith(f"hm3d_{split}"), data_sources.keys())
              )
              for split in ["train", "val", "minival", "example"]
          }
      )
  
      data_groups["hm3d"] = (
          data_groups["hm3d_val"]
          + data_groups["hm3d_train"]
          + data_groups["hm3d_minival"]
      )
  
      data_groups["ci_test_assets"].extend(data_groups["hm3d_example"])
function_tokens: def initialize_test_data_sources ( data_path ) : global data_sources global data_groups # dict keyed by uids with download specs for various individual test assets and datasets data_sources = { # "example_uid": { #   "source": the URL download link #   "package_name": the filename of the downloaded compressed package #   "download_pre_args": (optional)(wget) commands preceding filename #   "download_post_args": (optional)(wget) commands follow filename #   "link": symlink to the data directory pointing to the active version directory #   "version": data version tag # } "habitat_test_scenes" : { "source" : "http://dl.fbaipublicfiles.com/habitat/habitat-test-scenes_v1.0.zip" , "package_name" : "habitat-test-scenes_v1.0.zip" , "link" : data_path + "scene_datasets/habitat-test-scenes" , "version" : "1.0" , } , "habitat_test_pointnav_dataset" : { "source" : "http://dl.fbaipublicfiles.com/habitat/habitat-test-pointnav-dataset_v1.0.zip" , "package_name" : "habitat-test-pointnav-dataset_v1.0.zip" , "link" : data_path + "datasets/pointnav/habitat-test-scenes" , "version" : "1.0" , } , "habitat_example_objects" : { "source" : "http://dl.fbaipublicfiles.com/habitat/objects_v0.2.zip" , "package_name" : "objects_v0.2.zip" , "link" : data_path + "objects/example_objects" , "version" : "0.2" , } , "locobot_merged" : { "source" : "http://dl.fbaipublicfiles.com/habitat/locobot_merged_v0.2.zip" , "package_name" : "locobot_merged_v0.2.zip" , "link" : data_path + "objects/locobot_merged" , "version" : "0.2" , } , "mp3d_example_scene" : { "source" : "http://dl.fbaipublicfiles.com/habitat/mp3d/mp3d_example_v1.1.zip" , "package_name" : "mp3d_example_v1.1.zip" , "link" : data_path + "scene_datasets/mp3d_example" , "version" : "1.1" , } , "coda_scene" : { "source" : "https://dl.fbaipublicfiles.com/habitat/coda_v1.0.zip" , "package_name" : "coda_v1.0.zip" , "download_pre_args" : "--no-check-certificate " , "download_post_args" : " -O " + data_path + "coda_v1.0.zip" , "link" : data_path + "scene_datasets/coda" , "version" : "1.0" , } , "webxr_hand_demo" : { "source" : "https://dl.fbaipublicfiles.com/habitat/data/scene_datasets/webxr_hand_demo_data.zip" , "package_name" : "webxr_hand_demo_data.zip" , "link" : data_path + "webxr_hand_demo_dataset" , "version" : "1.0" , } , "replica_cad_dataset" : { "source" : "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/ReplicaCAD_dataset_v1.2.zip" , "package_name" : "ReplicaCAD_dataset_v1.2.zip" , "link" : data_path + "replica_cad" , "version" : "1.2" , } , "replica_cad_baked_lighting" : { "source" : "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/ReplicaCAD_baked_lighting_v1.1.zip" , "package_name" : "ReplicaCAD_baked_lighting_v1.1.zip" , "link" : data_path + "replica_cad_baked_lighting" , "version" : "1.1" , } , "ycb" : { "source" : "https://dl.fbaipublicfiles.com/habitat/ycb/hab_ycb_v1.1.zip" , "package_name" : "hab_ycb_v1.1.zip" , "link" : data_path + "objects/ycb" , "version" : "1.1" , } , "hab_fetch" : { "source" : "http://dl.fbaipublicfiles.com/habitat/hab_fetch_v1.0.zip" , "package_name" : "hab_fetch_v1.0.zip" , "link" : data_path + "robots/hab_fetch" , "version" : "1.0" , } , "rearrange_pick_dataset_v0" : { "source" : "https://dl.fbaipublicfiles.com/habitat/data/datasets/rearrange_pick/replica_cad/v0/rearrange_pick_replica_cad_v0.zip" , "package_name" : "rearrange_pick_replica_cad_v0.zip" , "link" : data_path + "datasets/rearrange_pick/replica_cad/v0" , "version" : "1.0" , } , "hab2_bench_assets" : { "source" : "https://dl.fbaipublicfiles.com/habitat/ReplicaCAD/hab2_bench_assets.zip" , "package_name" : "hab2_bench_assets.zip" , "link" : data_path + "hab2_bench_assets" , "version" : "1.0" , } , } data_sources . update ( { f"hm3d_{split}_{data_format}" : { "source" : "https://api.matterport.com/resources/habitat/hm3d-{split}-{data_format}.tar{ext}" . format ( ext = ".gz" if data_format == "obj+mtl" else "" , split = split , data_format = data_format , ) , "download_pre_args" : "--location" , "package_name" : "hm3d-{split}-{data_format}.tar{ext}" . format ( ext = ".gz" if data_format == "obj+mtl" else "" , split = split , data_format = data_format , ) , "link" : data_path + "scene_datasets/hm3d" , "version" : "1.0" , "version_dir" : "hm3d-{version}/hm3d" , "extract_postfix" : f"{split}" , "downloaded_file_list" : f"hm3d-{{version}}/{split}-{data_format}-files.json.gz" , "requires_auth" : True , "use_curl" : True , "post_extract_fn" : hm3d_train_configs_post if split == "train" and data_format == "configs" else None , } for split , data_format in itertools . product ( [ "minival" , "train" , "val" ] , [ "glb" , "obj+mtl" , "habitat" , "configs" ] , ) } ) data_sources . update ( { f"hm3d_example_{data_format}" : { "source" : "https://github.com/matterport/habitat-matterport-3dresearch/raw/main/example/hm3d-example-{data_format}.tar{ext}" . format ( ext = ".gz" if data_format == "obj+mtl" else "" , data_format = data_format , ) , "package_name" : "hm3d-example-{data_format}.tar{ext}" . format ( ext = ".gz" if data_format == "obj+mtl" else "" , data_format = data_format , ) , "link" : data_path + "scene_datasets/hm3d" , "version" : "1.0" , "version_dir" : "hm3d-{version}/hm3d" , "extract_postfix" : "example" , "downloaded_file_list" : f"hm3d-{{version}}/example-{data_format}-files.json.gz" , } for data_format in [ "glb" , "obj+mtl" , "habitat" , "configs" ] } ) # data sources can be grouped for batch commands with a new uid data_groups = { "ci_test_assets" : [ "habitat_test_scenes" , "habitat_test_pointnav_dataset" , "habitat_example_objects" , "locobot_merged" , "mp3d_example_scene" , "coda_scene" , "replica_cad_dataset" , "hab_fetch" , ] , "rearrange_task_assets" : [ "replica_cad_dataset" , "hab_fetch" , "ycb" , "rearrange_pick_dataset_v0" , ] , "hm3d_example" : [ "hm3d_example_habitat" , "hm3d_example_configs" ] , "hm3d_val" : [ "hm3d_val_habitat" , "hm3d_val_configs" ] , "hm3d_train" : [ "hm3d_train_habitat" , "hm3d_train_configs" ] , "hm3d_minival" : [ "hm3d_minival_habitat" , "hm3d_minival_configs" ] , "hm3d_full" : list ( filter ( lambda k : k . startswith ( "hm3d_" ) , data_sources . keys ( ) ) ) , } data_groups . update ( { f"hm3d_{split}_full" : list ( filter ( lambda k : k . startswith ( f"hm3d_{split}" ) , data_sources . keys ( ) ) ) for split in [ "train" , "val" , "minival" , "example" ] } ) data_groups [ "hm3d" ] = ( data_groups [ "hm3d_val" ] + data_groups [ "hm3d_train" ] + data_groups [ "hm3d_minival" ] ) data_groups [ "ci_test_assets" ] . extend ( data_groups [ "hm3d_example" ] )