Push

.gitattributes

@@ -1,87 +0,0 @@
-data/datasets/lilac/pile-of-law-constitutions/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-constitutions/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-constitutions/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/goal/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol1/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/piqa/sol2/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-atticus-contracts/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/choices/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/mmlu_professional_law/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/pile-of-law-r-legaladvice/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/open-asssistant-conversations/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/answers/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/context/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/squad_v2/question/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/imdb/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/hnsw.hnswlib.bin filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/hnsw.lookup.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/gte-small/spans.pkl filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/lang_detection/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/near_dup/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/pii/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/spacy_ner/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/datasets/lilac/wikitext-2-raw-v1/text/text_statistics/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -21,4 +21,4 @@ COPY /lilac ./lilac/
 
 COPY docker_start.sh docker_start.py ./
 
-CMD ./docker_start.sh
+CMD ["bash", "docker_start.sh"]

data/datasets/lilac/imdb/config.yml

@@ -1,31 +0,0 @@
-embeddings:
-- embedding: gte-small
-  path: text
-name: imdb
-namespace: local
-settings:
-  preferred_embedding: gte-small
-  ui:
-    media_paths:
-    - text
-signals:
-- path: text
-  signal:
-    signal_name: near_dup
-- path: text
-  signal:
-    signal_name: text_statistics
-- path: text
-  signal:
-    signal_name: lang_detection
-- path: text
-  signal:
-    signal_name: spacy_ner
-- path: text
-  signal:
-    signal_name: pii
-source:
-  dataset_name: imdb
-  source_name: huggingface
-tags:
-- machine-learning

data/datasets/lilac/imdb/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6fe90be23f86ca1e73b8a77a235344db822601f794a5643dca9d0d07c49ce3d8
-size 86160450

data/datasets/lilac/imdb/manifest.json

@@ -1,21 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "data_schema": {
-    "fields": {
-      "text": {
-        "dtype": "string"
-      },
-      "label": {
-        "dtype": "string"
-      },
-      "__hfsplit__": {
-        "dtype": "string"
-      },
-      "__rowid__": {
-        "dtype": "string"
-      }
-    }
-  }
-}

data/datasets/lilac/imdb/text/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7b8fc440b947f068966bc68c83ce4b5502b98e3aab928d58c295dc8a7b7b016c
-size 691432396

data/datasets/lilac/imdb/text/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d3ee2426c761b50025e02f2388fa65c51eb9705d6fd65f95d6502650214b8472
-size 10390867

data/datasets/lilac/imdb/text/gte-small/signal_manifest.json

@@ -1,35 +0,0 @@
-{
-  "files": [],
-  "parquet_id": "gte-small(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "gte-small": {
-            "repeated_field": {
-              "fields": {
-                "embedding": {
-                  "dtype": "embedding"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "signal_name": "gte-small"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "gte-small"
-  },
-  "enriched_path": [
-    "text"
-  ],
-  "vector_store": "hnsw"
-}

data/datasets/lilac/imdb/text/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:16417449057fc4304e098e037cb2b3e9a693570768a68bdca457d452adaee130
-size 7476546

data/datasets/lilac/imdb/text/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:adba6df16ec47b68625618b01a0b8d2cc65de572acb20eed561128b037fcdfd7
-size 3309315

data/datasets/lilac/imdb/text/lang_detection/signal_manifest.json

@@ -1,31 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "lang_detection(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "lang_detection": {
-            "dtype": "string",
-            "signal": {
-              "split_by_paragraph": false,
-              "signal_name": "lang_detection"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "split_by_paragraph": false,
-    "signal_name": "lang_detection"
-  },
-  "enriched_path": [
-    "text"
-  ]
-}

data/datasets/lilac/imdb/text/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:0c75a5f0d12b8b02671e99a2dd313d292610c9264f2302ad1877959430420079
-size 3915752

data/datasets/lilac/imdb/text/near_dup/signal_manifest.json

@@ -1,36 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "near_dup(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "near_dup": {
-            "fields": {
-              "cluster_id": {
-                "dtype": "uint32",
-                "categorical": true
-              }
-            },
-            "signal": {
-              "threshold": 0.85,
-              "signal_name": "near_dup"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "threshold": 0.85,
-    "signal_name": "near_dup"
-  },
-  "enriched_path": [
-    "text"
-  ]
-}

data/datasets/lilac/imdb/text/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2dfe71845bfd419a2a309b132456341bf317d63e8ccf8bc100835f1a20c81c5b
-size 3313701

data/datasets/lilac/imdb/text/pii/signal_manifest.json

@@ -1,45 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "pii(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "pii": {
-            "fields": {
-              "emails": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "ip_addresses": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "secrets": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              }
-            },
-            "signal": {
-              "signal_name": "pii"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "pii"
-  },
-  "enriched_path": [
-    "text"
-  ]
-}

data/datasets/lilac/imdb/text/spacy_ner/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:be39eb80c981da64a2ec5f7fe42c43ffb390b887a220e278a58cdde04e7824ef
-size 8479478

data/datasets/lilac/imdb/text/spacy_ner/signal_manifest.json

@@ -1,38 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "spacy_ner(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "spacy_ner": {
-            "repeated_field": {
-              "fields": {
-                "label": {
-                  "dtype": "string"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "model": "en_core_web_sm",
-              "signal_name": "spacy_ner"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "model": "en_core_web_sm",
-    "signal_name": "spacy_ner"
-  },
-  "enriched_path": [
-    "text"
-  ]
-}

data/datasets/lilac/imdb/text/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4919817d0c520fc5aacc9721f01a0e29fb4d2c89b6698865c9680c81c44ce920
-size 4403809

data/datasets/lilac/imdb/text/text_statistics/signal_manifest.json

@@ -1,59 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "text_statistics(text)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "text": {
-        "fields": {
-          "text_statistics": {
-            "fields": {
-              "num_characters": {
-                "dtype": "int32"
-              },
-              "readability": {
-                "dtype": "float32"
-              },
-              "log(type_token_ratio)": {
-                "dtype": "float32"
-              },
-              "frac_non_ascii": {
-                "dtype": "float32",
-                "bins": [
-                  [
-                    "Low",
-                    null,
-                    0.15
-                  ],
-                  [
-                    "Medium",
-                    0.15,
-                    0.3
-                  ],
-                  [
-                    "High",
-                    0.3,
-                    null
-                  ]
-                ]
-              }
-            },
-            "signal": {
-              "signal_name": "text_statistics"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "text_statistics"
-  },
-  "enriched_path": [
-    "text"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:df9d6e2f5df4b8693544f31ca78a9d1936a4caf47acc2babeb1cb766131b7636
-size 684360968

data/datasets/lilac/mmlu_professional_law/choices/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2081ce5d760026fe341e0553cd9e40747ca902e4e7edb851cb747f350f19bb0d
-size 11174465

data/datasets/lilac/mmlu_professional_law/choices/gte-small/signal_manifest.json

@@ -1,38 +0,0 @@
-{
-  "files": [],
-  "parquet_id": "gte-small(choices)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "choices": {
-        "repeated_field": {
-          "fields": {
-            "gte-small": {
-              "repeated_field": {
-                "fields": {
-                  "embedding": {
-                    "dtype": "embedding"
-                  }
-                },
-                "dtype": "string_span"
-              },
-              "signal": {
-                "signal_name": "gte-small"
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "gte-small"
-  },
-  "enriched_path": [
-    "choices",
-    "*"
-  ],
-  "vector_store": "hnsw"
-}

data/datasets/lilac/mmlu_professional_law/choices/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:02fb1662da21f33ea1429a0f9adf1301185da46f642a722717fe7c523314fa57
-size 11173475

data/datasets/lilac/mmlu_professional_law/choices/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:968d4f87c7b51b995d9e3a96423a06b91984e5ee4a47062cd53fe87cca5cafbe
-size 3469413

data/datasets/lilac/mmlu_professional_law/choices/lang_detection/signal_manifest.json

@@ -1,34 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "lang_detection(choices)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "choices": {
-        "repeated_field": {
-          "fields": {
-            "lang_detection": {
-              "dtype": "string",
-              "signal": {
-                "split_by_paragraph": false,
-                "signal_name": "lang_detection"
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "split_by_paragraph": false,
-    "signal_name": "lang_detection"
-  },
-  "enriched_path": [
-    "choices",
-    "*"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/choices/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:316f3be499fcbb960bc1e83a201838ca0b3047a71d8e1c302fe4e0d833a3bf90
-size 5544176

data/datasets/lilac/mmlu_professional_law/choices/near_dup/signal_manifest.json

@@ -1,39 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "near_dup(choices)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "choices": {
-        "repeated_field": {
-          "fields": {
-            "near_dup": {
-              "fields": {
-                "cluster_id": {
-                  "dtype": "uint32",
-                  "categorical": true
-                }
-              },
-              "signal": {
-                "threshold": 0.85,
-                "signal_name": "near_dup"
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "threshold": 0.85,
-    "signal_name": "near_dup"
-  },
-  "enriched_path": [
-    "choices",
-    "*"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/choices/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:7cb41d4e9d0d82bd824abfa733d5be3a599e011098c5d41ebadeb1166a15f722
-size 3393096

data/datasets/lilac/mmlu_professional_law/choices/pii/signal_manifest.json

@@ -1,48 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "pii(choices)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "choices": {
-        "repeated_field": {
-          "fields": {
-            "pii": {
-              "fields": {
-                "emails": {
-                  "repeated_field": {
-                    "dtype": "string_span"
-                  }
-                },
-                "ip_addresses": {
-                  "repeated_field": {
-                    "dtype": "string_span"
-                  }
-                },
-                "secrets": {
-                  "repeated_field": {
-                    "dtype": "string_span"
-                  }
-                }
-              },
-              "signal": {
-                "signal_name": "pii"
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "pii"
-  },
-  "enriched_path": [
-    "choices",
-    "*"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/choices/spacy_ner/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4b1255490f17c64f88b8b332c7df30060df612b9de11b17aaf6f70234c363e1e
-size 4080744

data/datasets/lilac/mmlu_professional_law/choices/spacy_ner/signal_manifest.json

@@ -1,41 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "spacy_ner(choices)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "choices": {
-        "repeated_field": {
-          "fields": {
-            "spacy_ner": {
-              "repeated_field": {
-                "fields": {
-                  "label": {
-                    "dtype": "string"
-                  }
-                },
-                "dtype": "string_span"
-              },
-              "signal": {
-                "model": "en_core_web_sm",
-                "signal_name": "spacy_ner"
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "model": "en_core_web_sm",
-    "signal_name": "spacy_ner"
-  },
-  "enriched_path": [
-    "choices",
-    "*"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/choices/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bc00a68e0f835e25b214d90e7e48251b39d748f1e836af713440cd0ea2517ead
-size 4634821

data/datasets/lilac/mmlu_professional_law/choices/text_statistics/signal_manifest.json

@@ -1,62 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "text_statistics(choices)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "choices": {
-        "repeated_field": {
-          "fields": {
-            "text_statistics": {
-              "fields": {
-                "num_characters": {
-                  "dtype": "int32"
-                },
-                "readability": {
-                  "dtype": "float32"
-                },
-                "log(type_token_ratio)": {
-                  "dtype": "float32"
-                },
-                "frac_non_ascii": {
-                  "dtype": "float32",
-                  "bins": [
-                    [
-                      "Low",
-                      null,
-                      0.15
-                    ],
-                    [
-                      "Medium",
-                      0.15,
-                      0.3
-                    ],
-                    [
-                      "High",
-                      0.3,
-                      null
-                    ]
-                  ]
-                }
-              },
-              "signal": {
-                "signal_name": "text_statistics"
-              }
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "text_statistics"
-  },
-  "enriched_path": [
-    "choices",
-    "*"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/config.yml

@@ -1,63 +0,0 @@
-embeddings:
-- embedding: gte-small
-  path:
-  - choices
-  - '*'
-- embedding: gte-small
-  path: question
-name: mmlu_professional_law
-namespace: local
-settings:
-  preferred_embedding: gte-small
-  ui:
-    media_paths:
-    - question
-    - - choices
-      - '*'
-signals:
-- path: question
-  signal:
-    signal_name: text_statistics
-- path: question
-  signal:
-    signal_name: pii
-- path: question
-  signal:
-    signal_name: near_dup
-- path:
-  - choices
-  - '*'
-  signal:
-    signal_name: text_statistics
-- path:
-  - choices
-  - '*'
-  signal:
-    signal_name: spacy_ner
-- path: question
-  signal:
-    signal_name: lang_detection
-- path:
-  - choices
-  - '*'
-  signal:
-    signal_name: near_dup
-- path:
-  - choices
-  - '*'
-  signal:
-    signal_name: pii
-- path:
-  - choices
-  - '*'
-  signal:
-    signal_name: lang_detection
-- path: question
-  signal:
-    signal_name: spacy_ner
-source:
-  config_name: professional_law
-  dataset_name: cais/mmlu
-  source_name: huggingface
-tags:
-- legal

data/datasets/lilac/mmlu_professional_law/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:65cd2771cf0bb88dbed9ad66ceaff472115f07c9dfea866c7e3f65b68392e745
-size 50699938

data/datasets/lilac/mmlu_professional_law/manifest.json

@@ -1,26 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "data_schema": {
-    "fields": {
-      "question": {
-        "dtype": "string"
-      },
-      "choices": {
-        "repeated_field": {
-          "dtype": "string"
-        }
-      },
-      "answer": {
-        "dtype": "string"
-      },
-      "__hfsplit__": {
-        "dtype": "string"
-      },
-      "__rowid__": {
-        "dtype": "string"
-      }
-    }
-  }
-}

data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.hnswlib.bin

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:3b02300405fccc3011294e15ee869933dd81578173435defbcb19e3b40a65e93
-size 771802212

data/datasets/lilac/mmlu_professional_law/question/gte-small/hnsw.lookup.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:f72169740d80ee2b2ea66589d7ebcc58c83381978a4640a27510c416a02bf6c7
-size 11296648

data/datasets/lilac/mmlu_professional_law/question/gte-small/signal_manifest.json

@@ -1,35 +0,0 @@
-{
-  "files": [],
-  "parquet_id": "gte-small(question)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "question": {
-        "fields": {
-          "gte-small": {
-            "repeated_field": {
-              "fields": {
-                "embedding": {
-                  "dtype": "embedding"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "signal_name": "gte-small"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "gte-small"
-  },
-  "enriched_path": [
-    "question"
-  ],
-  "vector_store": "hnsw"
-}

data/datasets/lilac/mmlu_professional_law/question/gte-small/spans.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9b51cad455e94b167bc9cf130c262ed1b143a8f386c7074a61983e01cd93d277
-size 7911602

data/datasets/lilac/mmlu_professional_law/question/lang_detection/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:bf6cf8cdc246ce4406599aec8782d3be02f2585f1fbad74173faf0ffcb453a49
-size 3361922

data/datasets/lilac/mmlu_professional_law/question/lang_detection/signal_manifest.json

@@ -1,31 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "lang_detection(question)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "question": {
-        "fields": {
-          "lang_detection": {
-            "dtype": "string",
-            "signal": {
-              "split_by_paragraph": false,
-              "signal_name": "lang_detection"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "split_by_paragraph": false,
-    "signal_name": "lang_detection"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/question/near_dup/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2c4139f699d1a248cf5378c442ef6f17970913394d5d0c79bd7c6e6801ab548a
-size 3697516

data/datasets/lilac/mmlu_professional_law/question/near_dup/signal_manifest.json

@@ -1,36 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "near_dup(question)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "question": {
-        "fields": {
-          "near_dup": {
-            "fields": {
-              "cluster_id": {
-                "dtype": "uint32",
-                "categorical": true
-              }
-            },
-            "signal": {
-              "threshold": 0.85,
-              "signal_name": "near_dup"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "threshold": 0.85,
-    "signal_name": "near_dup"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/question/pii/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:2735c4a2c5d40973652d369140533af74425db6dd753f8a25850d4efeee4928e
-size 3369080

data/datasets/lilac/mmlu_professional_law/question/pii/signal_manifest.json

@@ -1,45 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "pii(question)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "question": {
-        "fields": {
-          "pii": {
-            "fields": {
-              "emails": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "ip_addresses": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              },
-              "secrets": {
-                "repeated_field": {
-                  "dtype": "string_span"
-                }
-              }
-            },
-            "signal": {
-              "signal_name": "pii"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "pii"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/question/spacy_ner/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:e775b663f9a3b7c7ebdd31f9a860254dec31c18aa46c5a61820050d0556cbb0f
-size 9105982

data/datasets/lilac/mmlu_professional_law/question/spacy_ner/signal_manifest.json

@@ -1,38 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "spacy_ner(question)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "question": {
-        "fields": {
-          "spacy_ner": {
-            "repeated_field": {
-              "fields": {
-                "label": {
-                  "dtype": "string"
-                }
-              },
-              "dtype": "string_span"
-            },
-            "signal": {
-              "model": "en_core_web_sm",
-              "signal_name": "spacy_ner"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "model": "en_core_web_sm",
-    "signal_name": "spacy_ner"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}

data/datasets/lilac/mmlu_professional_law/question/text_statistics/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:995b3ac42907ea244d9cb04c68a4715af8ddb7d72dcced056bc58dc9a9f05e7e
-size 4389031

data/datasets/lilac/mmlu_professional_law/question/text_statistics/signal_manifest.json

@@ -1,59 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "parquet_id": "text_statistics(question)",
-  "data_schema": {
-    "fields": {
-      "__rowid__": {
-        "dtype": "string"
-      },
-      "question": {
-        "fields": {
-          "text_statistics": {
-            "fields": {
-              "num_characters": {
-                "dtype": "int32"
-              },
-              "readability": {
-                "dtype": "float32"
-              },
-              "log(type_token_ratio)": {
-                "dtype": "float32"
-              },
-              "frac_non_ascii": {
-                "dtype": "float32",
-                "bins": [
-                  [
-                    "Low",
-                    null,
-                    0.15
-                  ],
-                  [
-                    "Medium",
-                    0.15,
-                    0.3
-                  ],
-                  [
-                    "High",
-                    0.3,
-                    null
-                  ]
-                ]
-              }
-            },
-            "signal": {
-              "signal_name": "text_statistics"
-            }
-          }
-        }
-      }
-    }
-  },
-  "signal": {
-    "signal_name": "text_statistics"
-  },
-  "enriched_path": [
-    "question"
-  ]
-}