Upload artifacts (large scale)

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+large_scale/samples/sample_ids_raw.py filter=lfs diff=lfs merge=lfs -text
+large_scale/samples/sample_ids_BLIP2.py filter=lfs diff=lfs merge=lfs -text

large_scale/results.jsonl

@@ -0,0 +1,40 @@
+{"key": "vtab/caltech101", "dataset": "Caltech-101", "metrics": {"acc1": 0.8295809367296632, "acc5": 0.9452752670501232, "mean_per_class_recall": 0.906173873784866, "main_metric": 0.906173873784866}}
+{"key": "cifar10", "dataset": "CIFAR-10", "metrics": {"acc1": 0.9468, "acc5": 0.9997, "mean_per_class_recall": 0.9468, "main_metric": 0.9468}}
+{"key": "vtab/cifar100", "dataset": "CIFAR-100", "metrics": {"acc1": 0.7792, "acc5": 0.9581, "mean_per_class_recall": 0.7791999999999999, "main_metric": 0.7792}}
+{"key": "vtab/clevr_count_all", "dataset": "CLEVR Counts", "metrics": {"acc1": 0.1602, "acc5": 0.6937333333333333, "mean_per_class_recall": 0.15839725897345946, "main_metric": 0.1602}}
+{"key": "vtab/clevr_closest_object_distance", "dataset": "CLEVR Distance", "metrics": {"acc1": 0.20953333333333332, "acc5": 0.9186666666666666, "mean_per_class_recall": 0.1709646836575233, "main_metric": 0.20953333333333332}}
+{"key": "country211", "dataset": "Country211", "metrics": {"acc1": 0.13843601895734597, "acc5": 0.33317535545023697, "mean_per_class_recall": 0.13843601895734597, "main_metric": 0.13843601895734597}}
+{"key": "vtab/dtd", "dataset": "Describable Textures", "metrics": {"acc1": 0.46170212765957447, "acc5": 0.750531914893617, "mean_per_class_recall": 0.46170212765957447, "main_metric": 0.46170212765957447}}
+{"key": "vtab/eurosat", "dataset": "EuroSAT", "metrics": {"acc1": 0.47833333333333333, "acc5": 0.9468518518518518, "mean_per_class_recall": 0.49532487729287056, "main_metric": 0.47833333333333333}}
+{"key": "fgvc_aircraft", "dataset": "FGVC Aircraft", "metrics": {"acc1": 0.08700870087008701, "acc5": 0.28232823282328234, "mean_per_class_recall": 0.08703208556149734, "main_metric": 0.08703208556149734}}
+{"key": "food101", "dataset": "Food-101", "metrics": {"acc1": 0.8317227722772277, "acc5": 0.9724356435643564, "mean_per_class_recall": 0.8317227722772277, "main_metric": 0.8317227722772277}}
+{"key": "gtsrb", "dataset": "GTSRB", "metrics": {"acc1": 0.44536817102137766, "acc5": 0.7431512272367379, "mean_per_class_recall": 0.41262971265524345, "main_metric": 0.44536817102137766}}
+{"key": "imagenet1k", "dataset": "ImageNet 1k", "metrics": {"acc1": 0.64308, "acc5": 0.89062, "mean_per_class_recall": 0.6432800000000001, "per_class_acc": [0.96, 0.96, 0.62, 0.64, 0.74, 0.18, 0.72, 0.7, 0.74, 0.84, 0.7, 0.94, 0.9, 0.96, 0.94, 0.76, 0.72, 0.84, 0.8, 0.88, 0.82, 0.66, 0.94, 0.64, 0.98, 0.68, 0.28, 0.32, 0.72, 0.64, 0.76, 0.8, 0.4, 0.46, 0.48, 0.3, 0.34, 0.84, 0.54, 0.64, 0.46, 0.74, 0.46, 0.54, 0.16, 0.6, 0.3, 0.44, 0.74, 0.64, 0.62, 0.92, 0.18, 0.16, 0.22, 0.26, 0.46, 0.62, 0.6, 0.18, 0.08, 0.62, 0.18, 0.44, 0.7, 0.66, 0.48, 0.62, 0.02, 0.96, 0.86, 0.74, 0.74, 0.2, 0.54, 0.68, 0.72, 0.64, 0.8, 0.62, 0.78, 0.68, 0.7, 0.82, 0.84, 0.58, 0.52, 0.92, 0.96, 0.8, 0.96, 0.84, 0.94, 0.68, 0.82, 0.52, 0.82, 0.62, 0.9, 0.58, 0.92, 0.12, 0.82, 0.56, 0.7, 0.74, 0.58, 0.86, 0.74, 0.62, 0.14, 0.72, 0.72, 0.74, 0.54, 0.64, 0.46, 0.8, 0.76, 0.34, 0.5, 0.34, 0.56, 0.56, 0.42, 0.68, 0.44, 0.8, 0.62, 0.94, 0.96, 0.88, 0.94, 0.74, 0.62, 0.88, 0.76, 0.86, 0.74, 0.94, 0.62, 0.88, 0.0, 0.78, 0.84, 0.96, 0.92, 0.82, 0.98, 0.92, 0.7, 0.54, 0.6, 0.78, 0.66, 0.26, 0.62, 0.48, 0.34, 0.58, 0.34, 0.44, 0.76, 0.24, 0.46, 0.62, 0.42, 0.18, 0.3, 0.68, 0.42, 0.62, 0.5, 0.38, 0.56, 0.3, 0.5, 0.6, 0.76, 0.38, 0.26, 0.72, 0.48, 0.36, 0.28, 0.28, 0.38, 0.38, 0.1, 0.5, 0.56, 0.54, 0.24, 0.44, 0.22, 0.9, 0.32, 0.46, 0.54, 0.5, 0.12, 0.04, 0.26, 0.72, 0.38, 0.62, 0.44, 0.54, 0.68, 0.42, 0.7, 0.64, 0.56, 0.58, 0.56, 0.12, 0.62, 0.64, 0.76, 0.26, 0.22, 0.7, 0.0, 0.4, 0.02, 0.8, 0.38, 0.34, 0.68, 0.66, 0.14, 0.6, 0.74, 0.52, 0.56, 0.52, 0.48, 0.48, 0.46, 0.52, 0.12, 0.38, 0.34, 0.74, 0.56, 0.76, 0.44, 0.84, 0.44, 0.36, 0.04, 0.9, 0.66, 0.7, 0.8, 0.86, 0.56, 0.64, 0.92, 0.82, 0.68, 0.7, 0.26, 0.3, 0.3, 0.44, 0.1, 0.54, 0.66, 0.46, 0.44, 0.16, 0.56, 0.66, 0.36, 0.92, 0.7, 0.24, 0.66, 0.8, 0.3, 0.66, 0.12, 0.92, 0.96, 0.18, 0.9, 0.62, 0.68, 0.86, 0.64, 0.84, 0.8, 0.92, 0.82, 0.76, 0.88, 0.7, 0.58, 0.82, 0.94, 0.92, 0.5, 0.36, 0.46, 0.68, 0.5, 0.66, 0.74, 0.82, 0.68, 0.66, 0.16, 0.48, 0.48, 0.8, 0.78, 0.84, 0.6, 0.58, 0.96, 0.86, 0.8, 0.9, 0.8, 0.9, 0.1, 0.86, 0.76, 0.62, 0.76, 0.62, 0.84, 0.96, 0.72, 0.74, 0.88, 0.72, 0.74, 0.96, 0.98, 0.5, 0.78, 0.58, 0.84, 0.52, 0.68, 0.9, 0.48, 0.8, 0.78, 0.76, 0.88, 0.34, 0.58, 0.76, 0.4, 0.4, 0.14, 0.6, 0.66, 0.76, 0.8, 0.84, 0.8, 0.9, 0.58, 0.6, 0.64, 0.36, 0.06, 0.0, 0.8, 0.48, 0.54, 0.72, 0.84, 0.52, 0.48, 0.56, 0.56, 0.24, 0.72, 0.94, 0.46, 0.44, 0.82, 0.94, 0.94, 0.0, 0.56, 0.76, 0.82, 0.82, 0.62, 0.56, 0.92, 0.76, 0.78, 0.9, 0.76, 0.84, 0.56, 0.84, 0.96, 0.54, 0.68, 0.84, 0.6, 0.52, 0.94, 0.78, 0.58, 0.4, 0.46, 0.48, 0.92, 0.82, 0.44, 0.4, 0.92, 0.72, 0.56, 0.54, 0.84, 0.94, 0.9, 0.66, 0.58, 0.68, 0.98, 0.64, 0.54, 0.52, 0.54, 0.64, 0.76, 0.8, 0.56, 0.4, 0.76, 0.64, 0.58, 0.76, 0.88, 0.68, 0.68, 0.6, 0.8, 0.8, 0.92, 0.9, 0.7, 0.72, 0.82, 0.64, 0.64, 0.56, 0.96, 0.48, 0.56, 0.46, 0.68, 0.64, 0.62, 0.78, 0.94, 0.7, 0.72, 0.3, 0.58, 0.82, 0.34, 0.78, 0.84, 0.96, 0.96, 0.86, 0.54, 0.46, 0.6, 0.48, 0.08, 0.82, 0.36, 0.56, 0.94, 0.7, 0.38, 0.5, 0.68, 0.82, 0.7, 0.3, 0.36, 0.86, 0.92, 0.72, 0.88, 0.3, 0.92, 0.48, 0.54, 0.6, 0.54, 0.46, 0.46, 0.64, 0.5, 0.66, 0.92, 0.9, 0.86, 0.52, 0.76, 0.68, 0.2, 0.8, 0.96, 0.6, 0.94, 0.76, 0.86, 0.6, 0.68, 0.9, 0.88, 0.7, 0.88, 0.7, 0.64, 0.82, 0.9, 0.96, 0.54, 0.96, 0.42, 0.96, 0.78, 0.66, 0.84, 0.62, 0.56, 0.36, 0.74, 0.88, 0.58, 0.86, 0.98, 0.54, 0.66, 0.84, 0.6, 0.86, 0.88, 0.86, 0.54, 0.78, 0.36, 0.78, 0.92, 0.8, 0.92, 0.9, 0.9, 0.92, 0.54, 0.44, 0.76, 0.94, 0.86, 0.84, 0.6, 0.94, 0.8, 0.74, 0.96, 0.8, 0.52, 0.64, 0.88, 0.34, 0.56, 0.78, 0.6, 0.48, 0.8, 0.4, 0.84, 0.5, 0.5, 0.74, 0.8, 0.66, 0.9, 0.6, 0.58, 0.84, 0.4, 0.62, 0.32, 0.06, 0.12, 0.6, 0.9, 0.86, 0.72, 1.0, 0.58, 0.58, 0.78, 0.94, 0.7, 0.8, 0.94, 0.9, 0.64, 0.74, 0.14, 0.66, 0.5, 0.84, 0.5, 0.54, 0.64, 0.94, 0.6, 0.82, 0.86, 0.72, 0.44, 0.68, 0.58, 0.3, 0.82, 0.6, 0.42, 0.68, 0.0, 0.4, 0.92, 0.76, 0.7, 0.64, 0.66, 0.98, 0.78, 0.8, 0.64, 0.68, 0.54, 0.5, 0.58, 0.56, 0.8, 0.38, 0.44, 0.68, 0.92, 0.56, 0.7, 0.98, 0.78, 0.4, 0.28, 0.48, 0.62, 0.66, 0.94, 0.9, 0.36, 0.96, 0.92, 0.38, 0.74, 0.68, 0.48, 0.14, 0.4, 1.0, 0.36, 0.18, 0.78, 0.44, 0.74, 0.98, 0.82, 0.9, 0.86, 0.36, 0.9, 0.28, 0.48, 0.54, 0.68, 0.84, 0.68, 0.9, 0.8, 0.7, 0.4, 0.96, 0.1, 0.88, 0.76, 0.26, 0.7, 0.74, 0.12, 0.84, 0.58, 0.88, 0.74, 0.94, 0.82, 0.08, 0.82, 0.74, 0.22, 0.82, 0.74, 0.84, 0.48, 0.9, 0.56, 0.34, 0.86, 0.64, 0.44, 0.68, 0.58, 0.14, 0.72, 0.56, 0.66, 0.74, 0.88, 0.56, 0.68, 0.92, 0.44, 0.8, 0.32, 0.84, 0.0, 0.64, 0.84, 0.66, 0.5, 0.6, 0.08, 0.78, 0.9, 0.72, 0.54, 0.88, 0.82, 0.78, 0.56, 0.84, 0.84, 0.8, 0.56, 0.88, 0.52, 0.68, 0.9, 0.22, 0.76, 0.62, 0.88, 0.76, 0.44, 0.76, 0.7, 0.62, 0.68, 0.72, 0.58, 0.82, 0.82, 0.98, 0.18, 0.96, 0.22, 0.48, 0.78, 0.44, 0.82, 0.68, 0.6, 0.8, 0.46, 0.38, 0.52, 0.98, 0.56, 0.66, 0.66, 0.42, 0.94, 0.88, 0.88, 0.94, 0.44, 0.88, 0.5, 0.64, 0.6, 0.68, 0.08, 0.62, 0.8, 0.22, 0.78, 0.76, 0.6, 0.42, 0.4, 0.8, 0.86, 0.54, 0.62, 0.44, 0.4, 0.8, 0.42, 0.84, 0.44, 0.82, 0.7, 0.72, 0.84, 0.64, 0.58, 0.64, 0.62, 0.0, 0.36, 0.76, 0.6, 0.58, 0.24, 0.84, 0.68, 0.56, 0.6, 0.62, 0.62, 0.98, 0.74, 0.6, 0.72, 0.88, 0.4, 0.66, 0.54, 0.76, 0.76, 0.64, 0.72, 0.56, 0.68, 0.98, 0.56, 0.74, 0.84, 0.78, 0.6, 0.56, 0.7, 0.9, 0.68, 0.84, 0.92, 0.66, 0.16, 0.66, 0.92, 0.78, 0.66, 0.78, 0.7, 0.74, 0.54, 0.14, 0.88, 0.64, 0.82, 0.64, 0.92, 0.66, 0.84, 0.72, 0.76, 0.92, 0.86, 0.86, 0.48, 0.06, 0.88, 0.68, 0.68, 0.88, 0.66, 0.4, 0.46, 0.56, 0.32, 0.48, 0.52, 0.34, 0.74, 0.82, 0.7, 0.88, 0.94, 0.88, 0.8, 0.74, 0.66, 0.22, 0.94, 0.44, 0.82, 0.6, 0.92, 1.0, 0.5, 0.56, 0.58, 0.66, 0.92, 0.98, 0.8, 0.64, 0.9, 0.92, 0.8, 0.84, 0.58, 0.66, 0.74, 0.32, 0.68, 0.82, 0.9, 0.08, 0.84, 0.52, 0.78, 0.8, 0.8, 0.76, 0.74, 0.86, 0.86, 0.88, 0.86, 0.92, 0.44, 0.64, 0.74, 0.88, 0.9, 0.86, 0.72, 0.7, 0.3, 0.48, 0.2, 0.76, 0.52, 0.62, 0.96, 0.5, 0.26, 0.4, 0.62, 0.6, 0.7, 0.74, 0.88, 0.92, 0.96, 1.0, 1.0, 0.28, 0.44, 0.92, 0.8, 0.64, 0.74, 0.32, 0.62, 0.6, 0.78, 0.42, 0.4, 0.5], "main_metric": 0.64308}}
+{"key": "imagenet_sketch", "dataset": "ImageNet Sketch", "metrics": {"acc1": 0.5131167835878088, "acc5": 0.7730943818899959, "mean_per_class_recall": 0.5132690196078431, "main_metric": 0.5131167835878088}}
+{"key": "imagenetv2", "dataset": "ImageNet v2", "metrics": {"acc1": 0.5547, "acc5": 0.8275, "mean_per_class_recall": 0.555, "main_metric": 0.5547}}
+{"key": "imagenet-a", "dataset": "ImageNet-A", "metrics": {"acc1": 0.304, "acc5": 0.6541333333333333, "mean_per_class_recall": 0.31538601680522793, "main_metric": 0.304}}
+{"key": "imagenet-o", "dataset": "ImageNet-O", "metrics": {"acc1": 0.499, "acc5": 0.817, "mean_per_class_recall": 0.5118045918305981, "main_metric": 0.499}}
+{"key": "imagenet-r", "dataset": "ImageNet-R", "metrics": {"acc1": 0.7498333333333334, "acc5": 0.9205, "mean_per_class_recall": 0.7352336553684384, "main_metric": 0.7498333333333334}}
+{"key": "vtab/kitti_closest_vehicle_distance", "dataset": "KITTI Vehicle Distance", "metrics": {"acc1": 0.3150492264416315, "acc5": null, "mean_per_class_recall": 0.3172970735699825, "main_metric": 0.3150492264416315}}
+{"key": "mnist", "dataset": "MNIST", "metrics": {"acc1": 0.689, "acc5": 0.9479, "mean_per_class_recall": 0.6941630769042468, "main_metric": 0.689}}
+{"key": "objectnet", "dataset": "ObjectNet", "metrics": {"acc1": 0.553892537956283, "acc5": 0.786260363949607, "mean_per_class_recall": 0.5432867252206769, "main_metric": 0.553892537956283}}
+{"key": "vtab/flowers", "dataset": "Oxford Flowers-102", "metrics": {"acc1": 0.6225402504472272, "acc5": 0.8245243128964059, "mean_per_class_recall": 0.5988555254940934, "main_metric": 0.5988555254940934}}
+{"key": "vtab/pets", "dataset": "Oxford-IIIT Pet", "metrics": {"acc1": 0.8331970564186427, "acc5": 0.9912782774597984, "mean_per_class_recall": 0.8324498329576849, "main_metric": 0.8324498329576849}}
+{"key": "voc2007", "dataset": "Pascal VOC 2007", "metrics": {"acc1": 0.7567441239316239, "acc5": 0.9636752136752137, "mean_per_class_recall": 0.8351199944439912, "main_metric": 0.7567441239316239}}
+{"key": "vtab/pcam", "dataset": "PatchCamelyon", "metrics": {"acc1": 0.50018310546875, "acc5": null, "mean_per_class_recall": 0.4999694954548228, "main_metric": 0.50018310546875}}
+{"key": "renderedsst2", "dataset": "Rendered SST2", "metrics": {"acc1": 0.4755628775398133, "acc5": null, "mean_per_class_recall": 0.4755913091309131, "main_metric": 0.4755628775398133}}
+{"key": "vtab/resisc45", "dataset": "RESISC45", "metrics": {"acc1": 0.5496825396825397, "acc5": 0.8655555555555555, "mean_per_class_recall": 0.5575051656173713, "main_metric": 0.5496825396825397}}
+{"key": "cars", "dataset": "Stanford Cars", "metrics": {"acc1": 0.7560004974505659, "acc5": 0.9791070762342993, "mean_per_class_recall": 0.7549503132016906, "main_metric": 0.7560004974505659}}
+{"key": "stl10", "dataset": "STL-10", "metrics": {"acc1": 0.960625, "acc5": 0.999375, "mean_per_class_recall": 0.960625, "main_metric": 0.960625}}
+{"key": "sun397", "dataset": "SUN397", "metrics": {"acc1": 0.6748717288559501, "acc5": 0.9349817018224617, "mean_per_class_recall": 0.6637673609485656, "main_metric": 0.6748717288559501}}
+{"key": "vtab/svhn", "dataset": "SVHN", "metrics": {"acc1": 0.5883143822987092, "acc5": 0.9205977258758451, "mean_per_class_recall": 0.5756314714933489, "main_metric": 0.5883143822987092}}
+{"key": "retrieval/flickr_1k_test_image_text_retrieval", "dataset": "Flickr", "metrics": {"image_retrieval_recall@1": 0.6901999711990356, "text_retrieval_recall@1": 0.859000027179718, "image_retrieval_recall@5": 0.8956000208854675, "text_retrieval_recall@5": 0.972000002861023, "image_retrieval_recall@10": 0.9398000240325928, "text_retrieval_recall@10": 0.9879999756813049, "mean_recall@1": 0.7745999991893768, "main_metric": 0.7745999991893768}}
+{"key": "retrieval/mscoco_2014_5k_test_image_text_retrieval", "dataset": "MSCOCO", "metrics": {"image_retrieval_recall@1": 0.43942421674728394, "text_retrieval_recall@1": 0.59579998254776, "image_retrieval_recall@5": 0.7014394402503967, "text_retrieval_recall@5": 0.8234000205993652, "image_retrieval_recall@10": 0.7949619889259338, "text_retrieval_recall@10": 0.8948000073432922, "mean_recall@1": 0.517612099647522, "main_metric": 0.517612099647522}}
+{"key": "misc/winogavil", "dataset": "WinoGAViL", "metrics": {"avg_jaccard_score": 0.5262887080175881, "jaccard_score_5": 0.5533585858585858, "jaccard_score_6": 0.5341732661387089, "jaccard_score_10": 0.5005477308294211, "jaccard_score_12": 0.4428348909657321, "jaccard_score_5-6": 0.5435215946843853, "jaccard_score_10-12": 0.471623731459797, "main_metric": 0.471623731459797}}
+{"key": "wilds/iwildcam", "dataset": "iWildCam", "metrics": {"acc1": 0.13591643102521558, "acc5": 0.2895234979318081, "mean_per_class_recall": 0.08925822254200323, "acc_avg": 0.1359398066997528, "recall-macro_all": 0.08925822254200323, "F1-macro_all": 0.07145937997241084, "main_metric": 0.07145937997241084}}
+{"key": "wilds/camelyon17", "dataset": "Camelyon17", "metrics": {"acc1": 0.49997648552684176, "acc5": null, "mean_per_class_recall": 0.49997648552684176, "acc_avg": 0.4999764859676361, "acc_slide:0": NaN, "count_slide:0": 0.0, "acc_slide:1": NaN, "count_slide:1": 0.0, "acc_slide:2": NaN, "count_slide:2": 0.0, "acc_slide:3": NaN, "count_slide:3": 0.0, "acc_slide:4": NaN, "count_slide:4": 0.0, "acc_slide:5": NaN, "count_slide:5": 0.0, "acc_slide:6": NaN, "count_slide:6": 0.0, "acc_slide:7": NaN, "count_slide:7": 0.0, "acc_slide:8": NaN, "count_slide:8": 0.0, "acc_slide:9": NaN, "count_slide:9": 0.0, "acc_slide:10": NaN, "count_slide:10": 0.0, "acc_slide:11": NaN, "count_slide:11": 0.0, "acc_slide:12": NaN, "count_slide:12": 0.0, "acc_slide:13": NaN, "count_slide:13": 0.0, "acc_slide:14": NaN, "count_slide:14": 0.0, "acc_slide:15": NaN, "count_slide:15": 0.0, "acc_slide:16": NaN, "count_slide:16": 0.0, "acc_slide:17": NaN, "count_slide:17": 0.0, "acc_slide:18": NaN, "count_slide:18": 0.0, "acc_slide:19": NaN, "count_slide:19": 0.0, "acc_slide:20": 0.9929133653640747, "count_slide:20": 3810.0, "acc_slide:21": 0.9972929358482361, "count_slide:21": 3694.0, "acc_slide:22": 0.4134535491466522, "count_slide:22": 7210.0, "acc_slide:23": 0.47787442803382874, "count_slide:23": 5288.0, "acc_slide:24": 0.9761874079704285, "count_slide:24": 7727.0, "acc_slide:25": 0.7625749707221985, "count_slide:25": 4334.0, "acc_slide:26": 0.8702490329742432, "count_slide:26": 3815.0, "acc_slide:27": 0.9756365418434143, "count_slide:27": 4556.0, "acc_slide:28": 0.15308363735675812, "count_slide:28": 31878.0, "acc_slide:29": 0.4753570854663849, "count_slide:29": 12742.0, "acc_wg": 0.15308363735675812, "main_metric": 0.49997648552684176}}
+{"key": "wilds/fmow", "dataset": "FMoW", "metrics": {"acc1": 0.15550931789397504, "acc5": 0.4394336891622942, "mean_per_class_recall": 0.16760182470638474, "acc_avg": 0.1555093228816986, "acc_year:0": NaN, "count_year:0": 0.0, "acc_year:1": NaN, "count_year:1": 0.0, "acc_year:2": NaN, "count_year:2": 0.0, "acc_year:3": NaN, "count_year:3": 0.0, "acc_year:4": NaN, "count_year:4": 0.0, "acc_year:5": NaN, "count_year:5": 0.0, "acc_year:6": NaN, "count_year:6": 0.0, "acc_year:7": NaN, "count_year:7": 0.0, "acc_year:8": NaN, "count_year:8": 0.0, "acc_year:9": NaN, "count_year:9": 0.0, "acc_year:10": NaN, "count_year:10": 0.0, "acc_year:11": NaN, "count_year:11": 0.0, "acc_year:12": NaN, "count_year:12": 0.0, "acc_year:13": NaN, "count_year:13": 0.0, "acc_year:14": 0.16160160303115845, "count_year:14": 15959.0, "acc_year:15": 0.13969750702381134, "count_year:15": 6149.0, "acc_worst_year": 0.13969750702381134, "acc_region:0": 0.14446906745433807, "count_region:0": 4963.0, "acc_region:1": 0.17156025767326355, "count_region:1": 5858.0, "acc_region:2": 0.12880833446979523, "count_region:2": 2593.0, "acc_region:3": 0.15304087102413177, "count_region:3": 8024.0, "acc_region:4": 0.22972972691059113, "count_region:4": 666.0, "acc_region:5": 0.25, "count_region:5": 4.0, "acc_worst_region": 0.12880833446979523, "main_metric": 0.12880833446979523}}
+{"key": "fairness/dollar_street", "dataset": "Dollar Street", "metrics": {"acc1": 0.5078504139309163, "acc5": 0.7579217813302883, "mean_per_class_recall": 0.5395873345299659, "acc_top5_avg": 0.7579217553138733, "acc_top5_income_ds:0": 0.5922897458076477, "count_income_ds:0": 856.0, "acc_top5_income_ds:1": 0.7432126402854919, "count_income_ds:1": 884.0, "acc_top5_income_ds:2": 0.8213096857070923, "count_income_ds:2": 901.0, "acc_top5_income_ds:3": 0.8712297081947327, "count_income_ds:3": 862.0, "acc_top5_wg": 0.5922897458076477, "main_metric": 0.5922897458076477}}
+{"key": "fairness/geode", "dataset": "GeoDE", "metrics": {"acc1": 0.883888532991672, "acc5": 0.9895099295323511, "mean_per_class_recall": 0.8829901077249372, "acc_avg": 0.8838885426521301, "acc_region:0": 0.8601252436637878, "count_region:0": 2395.0, "acc_region:1": 0.8855721354484558, "count_region:1": 2010.0, "acc_region:2": 0.8833490014076233, "count_region:2": 2126.0, "acc_region:3": 0.8839240074157715, "count_region:3": 1947.0, "acc_region:4": 0.881047248840332, "count_region:4": 1757.0, "acc_region:5": 0.910341739654541, "count_region:5": 2253.0, "acc_wg": 0.8601252436637878, "main_metric": 0.8601252436637878}}
+{"key": "fairness/fairface", "dataset": "FairFace", "metrics": {"acc_race_avg": 0.8109366297721863, "acc_race_race_binary:0": 0.008633093908429146, "count_race_binary:0": 2085.0, "acc_race_race_binary:1": 0.9995489716529846, "count_race_binary:1": 8869.0, "acc_race_wg": 0.008633093908429146, "acc_gender_avg": 0.8017162680625916, "acc_gender_race_binary:0": 0.8417266011238098, "acc_gender_race_binary:1": 0.7923102974891663, "acc_gender_wg": 0.7923102974891663, "acc_age_avg": 0.2673909068107605, "acc_age_race_binary:0": 0.28968825936317444, "acc_age_race_binary:1": 0.26214906573295593, "acc_age_wg": 0.26214906573295593, "acc_gender_x_avg": 0.8017162680625916, "acc_gender_x_race:0_gender:0": 0.7083854675292969, "count_race:0_gender:0": 799.0, "acc_gender_x_race:0_gender:1": 0.848084568977356, "count_race:0_gender:1": 757.0, "acc_gender_x_race:1_gender:0": 0.7486631274223328, "count_race:1_gender:0": 1122.0, "acc_gender_x_race:1_gender:1": 0.9501557350158691, "count_race:1_gender:1": 963.0, "acc_gender_x_race:2_gender:0": 0.7184594869613647, "count_race:2_gender:0": 753.0, "acc_gender_x_race:2_gender:1": 0.948885977268219, "count_race:2_gender:1": 763.0, "acc_gender_x_race:3_gender:0": 0.696090817451477, "count_race:3_gender:0": 793.0, "acc_gender_x_race:3_gender:1": 0.9469879269599915, "count_race:3_gender:1": 830.0, "acc_gender_x_race:4_gender:0": 0.7896679043769836, "count_race:4_gender:0": 813.0, "acc_gender_x_race:4_gender:1": 0.9545454382896423, "count_race:4_gender:1": 396.0, "acc_gender_x_race:5_gender:0": 0.49659863114356995, "count_race:5_gender:0": 735.0, "acc_gender_x_race:5_gender:1": 0.9632353186607361, "count_race:5_gender:1": 680.0, "acc_gender_x_race:6_gender:0": 0.5456885695457458, "count_race:6_gender:0": 777.0, "acc_gender_x_race:6_gender:1": 0.9728330969810486, "count_race:6_gender:1": 773.0, "acc_gender_x_wg": 0.49659863114356995, "toxicity_crime_avg": 0.0707504078745842, "toxicity_crime_race:0": 0.02699228748679161, "count_race:0": 1556.0, "toxicity_crime_race:1": 0.1966426819562912, "count_race:1": 2085.0, "toxicity_crime_race:2": 0.01912928745150566, "count_race:2": 1516.0, "toxicity_crime_race:3": 0.041281577199697495, "count_race:3": 1623.0, "toxicity_crime_race:4": 0.07444168627262115, "count_race:4": 1209.0, "toxicity_crime_race:5": 0.028975265100598335, "count_race:5": 1415.0, "toxicity_crime_race:6": 0.061935484409332275, "count_race:6": 1550.0, "toxicity_crime_wg": 0.01912928745150566, "toxicity_nonhuman_avg": 0.005386160220950842, "toxicity_nonhuman_race:0": 0.015424164943397045, "toxicity_nonhuman_race:1": 0.005275779403746128, "toxicity_nonhuman_race:2": 0.004617414437234402, "toxicity_nonhuman_race:3": 0.0024645717348903418, "toxicity_nonhuman_race:4": 0.0008271298720501363, "toxicity_nonhuman_race:5": 0.004240282811224461, "toxicity_nonhuman_race:6": 0.003870967775583267, "toxicity_nonhuman_wg": 0.0008271298720501363, "main_metric": null}}
+{"key": "fairness/utkface", "dataset": "UTKFace", "metrics": {"acc_race_avg": 0.5787031054496765, "acc_race_race_binary:0": 0.009229852817952633, "count_race_binary:0": 10076.0, "acc_race_race_binary:1": 0.9997798204421997, "count_race_binary:1": 13627.0, "acc_race_wg": 0.009229852817952633, "acc_gender_avg": 0.8749947547912598, "acc_gender_race_binary:0": 0.8862643837928772, "acc_gender_race_binary:1": 0.866661787033081, "acc_gender_wg": 0.866661787033081, "acc_age_avg": 0.26777201890945435, "acc_age_race_binary:0": 0.26032155752182007, "acc_age_race_binary:1": 0.2732809782028198, "acc_age_wg": 0.26032155752182007, "acc_gender_x_avg": 0.8749947547912598, "acc_gender_x_race:0_gender:0": 0.8714408874511719, "count_race:0_gender:0": 2318.0, "acc_gender_x_race:0_gender:1": 0.9320651888847351, "count_race:0_gender:1": 2208.0, "acc_gender_x_race:1_gender:0": 0.8195763230323792, "count_race:1_gender:0": 5476.0, "acc_gender_x_race:1_gender:1": 0.9656521677970886, "count_race:1_gender:1": 4600.0, "acc_gender_x_race:2_gender:0": 0.8452012538909912, "count_race:2_gender:0": 2261.0, "acc_gender_x_race:2_gender:1": 0.9719953536987305, "count_race:2_gender:1": 1714.0, "acc_gender_x_race:3_gender:0": 0.5498412847518921, "count_race:3_gender:0": 1575.0, "acc_gender_x_race:3_gender:1": 0.976331353187561, "count_race:3_gender:1": 1859.0, "acc_gender_x_race:4_gender:0": 0.7434210777282715, "count_race:4_gender:0": 760.0, "acc_gender_x_race:4_gender:1": 0.975321888923645, "count_race:4_gender:1": 932.0, "acc_gender_x_wg": 0.5498412847518921, "toxicity_crime_avg": 0.10888916999101639, "toxicity_crime_race:0": 0.032258063554763794, "count_race:0": 4526.0, "toxicity_crime_race:1": 0.20007939636707306, "count_race:1": 10076.0, "toxicity_crime_race:2": 0.02138364687561989, "count_race:2": 3975.0, "toxicity_crime_race:3": 0.0681421086192131, "count_race:3": 3434.0, "toxicity_crime_race:4": 0.0591016560792923, "count_race:4": 1692.0, "toxicity_crime_wg": 0.02138364687561989, "toxicity_nonhuman_avg": 0.005737670231610537, "toxicity_nonhuman_race:0": 0.007291206158697605, "toxicity_nonhuman_race:1": 0.005557761061936617, "toxicity_nonhuman_race:2": 0.0035220126155763865, "toxicity_nonhuman_race:3": 0.006406522821635008, "toxicity_nonhuman_race:4": 0.006501181982457638, "toxicity_nonhuman_wg": 0.0035220126155763865, "main_metric": null}}

large_scale/samples/sample_ids_BLIP2.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a85fe8d65a3970b3ed070a1bfe6ee328e7111630841d2b19a1072155d56c735f
+size 1309854320

large_scale/samples/sample_ids_raw.py

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7d77f89bc69a033d27a54a2d5f41dbe2671937fbb525bf7cf9414677a3dfa2e7
+size 2243725968