[WIP] Upload folder using huggingface_hub (multi-commit b33a1f5eb02e9046ee6631d7578b49f7afa8168ea7fea4605369684398ceadc2)

.env

@@ -0,0 +1,40 @@
+# To overwrite these variables, create a .env.local file
+
+# The path to the directory where the data will be downloaded on machine
+LILAC_DATA_PATH=./data
+
+# Set to 1 for duckdb to use views instead of materialized tables (lower memory usage, but slower).
+DUCKDB_USE_VIEWS=0
+
+# Set to true to enable read-only mode, disabling the ability to add datasets & compute dataset
+# signals.
+# LILAC_AUTH_ENABLED=true
+
+# Variables that can be set in .env.local
+#
+# Get key from https://dashboard.cohere.ai/api-keys
+# COHERE_API_KEY=
+
+# GCS_REGION=
+# GCS_ACCESS_KEY=
+# GCS_SECRET_KEY=
+
+# Get key from https://platform.openai.com/account/api-keys
+# OPENAI_API_KEY=
+# Get key from https://makersuite.google.com/app/apikey
+# PALM_API_KEY=
+
+# HuggingFace demos: machine that uploads to HuggingFace.
+
+# For authenticating with HuggingFace to deploy to a Space.
+# HF_USERNAME=
+# The default repo to deploy to for a staging demo. Can be overridden by a command line flag.
+# HF_STAGING_DEMO_REPO='HF_ORG/HF_REPO_NAME'
+
+# For Google-login. This is generated from the Google Cloud Console for a web client.
+# See: https://developers.google.com/identity/protocols/oauth2
+GOOGLE_CLIENT_ID='279475920249-i8llm8vbos1vj5m1qocir8narb3r0enu.apps.googleusercontent.com'
+# The client secret of the above client.
+# GOOGLE_CLIENT_SECRET=
+# A random string for oauth sessions.
+# LILAC_OAUTH_SECRET_KEY=

.env.demo

@@ -0,0 +1,4 @@
+LILAC_DATA_PATH='/data'
+HF_HOME='/data/.huggingface'
+TRANSFORMERS_CACHE='/data/.cache'
+XDG_CACHE_HOME='/data/.cache'

.gitattributes

@@ -1,99 +0,0 @@
-data/.cache/lilac/concept/lilac/profanity/gte-small.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/question/cohere.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/question/gte-base.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/question/openai.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/question/palm.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/toxicity/gte-small.pkl filter=lfs diff=lfs merge=lfs -text
-dist/lilac-0.0.20-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
-dist/lilac-0.1.1-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/profanity/gte-base.pkl filter=lfs diff=lfs merge=lfs -text
-dist/lilac-0.1.3-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
-dist/lilac-0.1.4-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
-dist/lilac-1337.0.0-py3-none-any.whl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_flat.00000-of-00001.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/num_conversations.00000-of-00001.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00010-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00005-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00002-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00003-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00004-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00011-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00006-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00001-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00009-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00008-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00000-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00007-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations_map.00000-of-00001.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/open-asssistant-conversations/text/pii.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len2.00000-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len2.00001-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len3.00002-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len3.00000-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len2.00002-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len3.00001-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len.00002-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len.00000-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-1M/question/len.00001-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca-10k/upper_question.00000-of-00001.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/lilac/OpenOrca-100k/upper_question.00002-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/lilac/OpenOrca-100k/upper_question.00000-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/lilac/OpenOrca-100k/upper_question.00001-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00010-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00005-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00002-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00002-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00005-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00003-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00004-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00004-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00011-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00003-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00001-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00006-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00006-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00001-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00009-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00008-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00007-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00000-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00008-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00000-of-00009.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00007-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00001-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00000-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00000-of-00001.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00001-of-00002.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00002-of-00003.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text_len.00000-of-00002.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/label/text_statistics.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00007-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00008-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00000-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00001-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00009-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00006-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00004-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00011-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00003-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00002-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00010-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer_formatted.00005-of-00012.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/lilac/OpenOrca-100k/question2.00001-of-00002.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/lilac/OpenOrca-100k/question2.00000-of-00002.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb/text/span.00000-of-00001.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/answer/markdown_extractor.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/glaive/question/markdown_extractor.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenHermes-2.5/conversations/\*/value/markdown_extractor.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/OpenOrca/response/markdown_code_block.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/imdb7/text/text_statistics.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/lilac/OpenOrca-100k/question/text_statistics.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/Capybara/input.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/local/Capybara/conversation/\*/input__cluster.jsonl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/prompt__cluster-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/prompt-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/hf_download/lilacai/nikhil_staging-local-Capybara/local/Capybara/data-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/hf_download/lilacai/nikhil_staging-local-Capybara/local/Capybara/input-00000-of-00001.parquet filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/question/nomic-embed-1.5-768.pkl filter=lfs diff=lfs merge=lfs -text
-data/.cache/lilac/concept/lilac/toxicity/openai.pkl filter=lfs diff=lfs merge=lfs -text

Dockerfile

@@ -1,48 +1,29 @@
-FROM python:3.11-slim-bullseye
+# NOTE: When we upgrade to 3.11 we can use a slimmer docker image which comes with gcc.
+FROM python:3.9-bullseye
 
 # Allow statements and log messages to immediately appear in the Knative logs
 ENV PYTHONUNBUFFERED True
-# Fixes "invalid instruction" runtime error on AMD machines (specifically HF Upgraded CPU Space).
-ENV HNSWLIB_NO_NATIVE 1
-
-# Adds GCC and other build tools so we can compile hnswlib and other native/C++ deps.
-RUN apt-get update --fix-missing && apt-get install -y --fix-missing build-essential && \
-  rm -rf /var/lib/apt/lists/*
-
-# See: https://huggingface.co/docs/hub/spaces-sdks-docker#permissions
-RUN useradd -m -u 1000 user
-USER user
-ENV HOME=/home/user \
-  PATH=/home/user/.local/bin:$PATH
 
 # Set the working directory in the container.
-WORKDIR $HOME/app
-
-# Install the dependencies. This will look in ./dist for any wheels that match lilac. If they are
-# not found, it will use the public pip package.
-
-# Pip install lilac[all] and dependencies before trying to install the local image. This allows us
-# to get cache hits on dependency installations when using a local wheel. When using the public pip
-# package, the second call will be a no-op.
-RUN python -m pip install lilac[all]
-
-# Install from the local wheel inside ./dist. This will be a no-op if the wheel is not found.
-COPY --chown=user /dist ./dist/
-RUN python -m pip install --find-links=dist --upgrade lilac[all]
+WORKDIR /server
 
-# Install the huggingface hub, used to download files.
-RUN pip install huggingface_hub
+# Install the dependencies. This requires exporting requirements.txt from poetry first, which
+# happens from ./build_docker.sh.
+COPY requirements.txt .
+RUN pip install --no-cache-dir -r requirements.txt
 
-# Copy the README so we can read the datasets from the HuggingFace config.
-COPY --chown=user README.md .
-# Copy the license just in case.
-COPY --chown=user LICENSE .
+COPY .env .
+COPY .env.demo .
+COPY LICENSE .
 
-COPY --chown=user docker_start.sh ./
+# Copy python files.
+COPY /lilac ./lilac/
 
-# Make a local data directory for non-persistent storage demos.
-RUN mkdir -p ./data
-RUN chown -R user ./data
+# Copy the data files. We use glob so docker copy won't fail if the directory doesn't exist.
+COPY /dat[a] ./data/
 
-EXPOSE 5432
-CMD ["bash", "docker_start.sh"]
+CMD [ \
+  "gunicorn", "lilac.server:app", \
+  "--bind", "0.0.0.0:5432", \
+  "-k", "uvicorn.workers.UvicornWorker" \
+  ]

LICENSE

@@ -2,142 +2,180 @@
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
-TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-1.  Definitions.
-
-    "License" shall mean the terms and conditions for use, reproduction, and distribution as defined
-    by Sections 1 through 9 of this document.
-
-    "Licensor" shall mean the copyright owner or entity authorized by the copyright owner that is
-    granting the License.
-
-    "Legal Entity" shall mean the union of the acting entity and all other entities that control,
-    are controlled by, or are under common control with that entity. For the purposes of this
-    definition, "control" means (i) the power, direct or indirect, to cause the direction or
-    management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent
-    (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity.
-
-    "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by
-    this License.
-
-    "Source" form shall mean the preferred form for making modifications, including but not limited
-    to software source code, documentation source, and configuration files.
-
-    "Object" form shall mean any form resulting from mechanical transformation or translation of a
-    Source form, including but not limited to compiled object code, generated documentation, and
-    conversions to other media types.
-
-    "Work" shall mean the work of authorship, whether in Source or Object form, made available under
-    the License, as indicated by a copyright notice that is included in or attached to the work (an
-    example is provided in the Appendix below).
-
-    "Derivative Works" shall mean any work, whether in Source or Object form, that is based on (or
-    derived from) the Work and for which the editorial revisions, annotations, elaborations, or
-    other modifications represent, as a whole, an original work of authorship. For the purposes of
-    this License, Derivative Works shall not include works that remain separable from, or merely
-    link (or bind by name) to the interfaces of, the Work and Derivative Works thereof.
-
-    "Contribution" shall mean any work of authorship, including the original version of the Work and
-    any modifications or additions to that Work or Derivative Works thereof, that is intentionally
-    submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or
-    Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this
-    definition, "submitted" means any form of electronic, verbal, or written communication sent to
-    the Licensor or its representatives, including but not limited to communication on electronic
-    mailing lists, source code control systems, and issue tracking systems that are managed by, or
-    on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding
-    communication that is conspicuously marked or otherwise designated in writing by the copyright
-    owner as "Not a Contribution."
-
-    "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a
-    Contribution has been received by Licensor and subsequently incorporated within the Work.
-
-2.  Grant of Copyright License. Subject to the terms and conditions of this License, each
-    Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free,
-    irrevocable copyright license to reproduce, prepare Derivative Works of, publicly display,
-    publicly perform, sublicense, and distribute the Work and such Derivative Works in Source or
-    Object form.
-
-3.  Grant of Patent License. Subject to the terms and conditions of this License, each Contributor
-    hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-    (except as stated in this section) patent license to make, have made, use, offer to sell, sell,
-    import, and otherwise transfer the Work, where such license applies only to those patent claims
-    licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or
-    by combination of their Contribution(s) with the Work to which such Contribution(s) was
-    submitted. If You institute patent litigation against any entity (including a cross-claim or
-    counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work
-    constitutes direct or contributory patent infringement, then any patent licenses granted to You
-    under this License for that Work shall terminate as of the date such litigation is filed.
-
-4.  Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof
-    in any medium, with or without modifications, and in Source or Object form, provided that You
-    meet the following conditions:
-
-    (a) You must give any other recipients of the Work or Derivative Works a copy of this License;
-    and
-
-    (b) You must cause any modified files to carry prominent notices stating that You changed the
-    files; and
-
-    (c) You must retain, in the Source form of any Derivative Works that You distribute, all
-    copyright, patent, trademark, and attribution notices from the Source form of the Work,
-    excluding those notices that do not pertain to any part of the Derivative Works; and
-
-    (d) If the Work includes a "NOTICE" text file as part of its distribution, then any Derivative
-    Works that You distribute must include a readable copy of the attribution notices contained
-    within such NOTICE file, excluding those notices that do not pertain to any part of the
-    Derivative Works, in at least one of the following places: within a NOTICE text file distributed
-    as part of the Derivative Works; within the Source form or documentation, if provided along with
-    the Derivative Works; or, within a display generated by the Derivative Works, if and wherever
-    such third-party notices normally appear. The contents of the NOTICE file are for informational
-    purposes only and do not modify the License. You may add Your own attribution notices within
-    Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the
-    Work, provided that such additional attribution notices cannot be construed as modifying the
-    License.
-
-    You may add Your own copyright statement to Your modifications and may provide additional or
-    different license terms and conditions for use, reproduction, or distribution of Your
-    modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and
-    distribution of the Work otherwise complies with the conditions stated in this License.
-
-5.  Submission of Contributions. Unless You explicitly state otherwise, any Contribution
-    intentionally submitted for inclusion in the Work by You to the Licensor shall be under the
-    terms and conditions of this License, without any additional terms or conditions.
-    Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate
-    license agreement you may have executed with Licensor regarding such Contributions.
-
-6.  Trademarks. This License does not grant permission to use the trade names, trademarks, service
-    marks, or product names of the Licensor, except as required for reasonable and customary use in
-    describing the origin of the Work and reproducing the content of the NOTICE file.
-
-7.  Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor
-    provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT
-    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation,
-    any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-    PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or
-    redistributing the Work and assume any risks associated with Your exercise of permissions under
-    this License.
-
-8.  Limitation of Liability. In no event and under no legal theory, whether in tort (including
-    negligence), contract, or otherwise, unless required by applicable law (such as deliberate and
-    grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for
-    damages, including any direct, indirect, special, incidental, or consequential damages of any
-    character arising as a result of this License or out of the use or inability to use the Work
-    (including but not limited to damages for loss of goodwill, work stoppage, computer failure or
-    malfunction, or any and all other commercial damages or losses), even if such Contributor has
-    been advised of the possibility of such damages.
-
-9.  Accepting Warranty or Additional Liability. While redistributing the Work or Derivative Works
-    thereof, You may choose to offer, and charge a fee for, acceptance of support, warranty,
-    indemnity, or other liability obligations and/or rights consistent with this License. However,
-    in accepting such obligations, You may act only on Your own behalf and on Your sole
-    responsibility, not on behalf of any other Contributor, and only if You agree to indemnify,
-    defend, and hold each Contributor harmless for any liability incurred by, or claims asserted
-    against, such Contributor by reason of your accepting any such warranty or additional liability.
-
-END OF TERMS AND CONDITIONS
-
-APPENDIX: How to apply the Apache License to your work.
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
 
       To apply the Apache License to your work, attach the following
       boilerplate notice, with the fields enclosed by brackets "[]"
@@ -148,14 +186,16 @@ APPENDIX: How to apply the Apache License to your work.
       same "printed page" as the copyright notice for easier
       identification within third-party archives.
 
-Copyright 2024 Lilac AI Inc.
+   Copyright 2023 Lilac AI Inc.
 
-Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in
-compliance with the License. You may obtain a copy of the License at
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
 
        http://www.apache.org/licenses/LICENSE-2.0
 
-Unless required by applicable law or agreed to in writing, software distributed under the License is
-distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-implied. See the License for the specific language governing permissions and limitations under the
-License.
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

README.md

@@ -1,10 +1,8 @@
 ---
-title: Lilac
-emoji: "\U0001F337"
+title: Lilac Blueprint
+emoji: 🌷
 colorFrom: purple
 colorTo: purple
 sdk: docker
 app_port: 5432
-datasets: []
-
 ---

data/.cache/lilac/concept/lilac/profanity/gte-base.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:986fa54683a7222e26bc838ab908ad6a0076809d26e77517143067c08be9bce6
-size 3307642

data/.cache/lilac/concept/lilac/profanity/gte-small.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:1677935b166c260856acd2e53a65b81693d0849010eac88f1ea923b2be7d614c
-size 1676204

data/.cache/lilac/concept/lilac/question/cohere.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:9768c28d6ed72e4a1a5819fef4157fb1f30a50f1e165bfcdd87d0fa761146902
-size 6254174

data/.cache/lilac/concept/lilac/question/gte-base.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:d2ae5bf4275be11be99cb2e90c03e35c9d2749efc3b34a2d1db1e9f0c99325d6
-size 1194925

data/.cache/lilac/concept/lilac/question/nomic-embed-1.5-768.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:c0ca65cf1fb5892e7073cb2882c7d5362ac549574d3989abea5191c1bc12a601
-size 1194959

data/.cache/lilac/concept/lilac/question/openai.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:4a14c6df6924f45391654fe78dee8cf996de3abb8acf8ca0f81a65814572d493
-size 2362432

data/.cache/lilac/concept/lilac/question/palm.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:cce86748bca57bd328f68b97ee80b4e3343ee4008d8951c5b061b6dd9335df7e
-size 1194921

data/.cache/lilac/concept/lilac/toxicity/gte-small.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:b6a074a3ac60cc9bfb82c4bf19d0e8c8d3837cb2b68b97efe8960c16675477f1
-size 1886420

data/.cache/lilac/concept/lilac/toxicity/openai.pkl

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:722c5141fb02f101c45b0530ae2de38685716b03b8bd52592a41f6e1c62dafc9
-size 7384504

data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/data-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:966e7068b352c73613629a44cd767c9eacc2a5c907a9addb97e9167ff2cc96e4
-size 495488044

data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/manifest.json

@@ -1,117 +0,0 @@
-{
-  "files": [
-    "data-00000-of-00001.parquet"
-  ],
-  "data_schema": {
-    "fields": {
-      "source": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "title": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "conversations": {
-        "repeated_field": {
-          "fields": {
-            "from": {
-              "dtype": {
-                "type": "string"
-              }
-            },
-            "value": {
-              "dtype": {
-                "type": "string"
-              }
-            },
-            "weight": {
-              "dtype": {
-                "type": "float64"
-              }
-            }
-          }
-        }
-      },
-      "id": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "category": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "model_name": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "language": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "topic": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "model": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "avatarUrl": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "skip_prompt_formatting": {
-        "dtype": {
-          "type": "boolean"
-        }
-      },
-      "idx": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "hash": {
-        "repeated_field": {
-          "dtype": {
-            "type": "int64"
-          }
-        }
-      },
-      "system_prompt": {
-        "dtype": {
-          "type": "string"
-        }
-      },
-      "custom_instruction": {
-        "dtype": {
-          "type": "boolean"
-        }
-      },
-      "views": {
-        "dtype": {
-          "type": "int64"
-        }
-      },
-      "__hfsplit__": {
-        "dtype": {
-          "type": "string"
-        }
-      }
-    }
-  },
-  "source": {
-    "dataset_name": "teknium/OpenHermes-2.5",
-    "load_from_disk": false,
-    "source_name": "huggingface"
-  }
-}

data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/prompt-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:51a7ece74402990921d852d5993a6a9deb8fc1867d731b43c1281dea8dcd98a5
-size 353857351

data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/prompt.map_manifest.json

@@ -1,24 +0,0 @@
-{
-  "files": [
-    "prompt-00000-of-00001.parquet"
-  ],
-  "parquet_id": "map.prompt",
-  "data_schema": {
-    "fields": {
-      "prompt": {
-        "dtype": {
-          "type": "string"
-        },
-        "map": {
-          "fn_name": "get_prompt",
-          "input_path": [
-            "conversations"
-          ],
-          "fn_source": "def get_prompt(conversations):\n  result = []\n  for msg in conversations:\n    if msg['from'] == 'human':\n      result.append(msg['value'])\n  return '\\n'.join(result)\n",
-          "date_created": "2024-01-03T13:19:00.151579"
-        }
-      }
-    }
-  },
-  "py_version": "0.1.26"
-}

data/.cache/lilac/hf_download/lilacai/hermes-clusters-local-OpenHermes-2.5/local/OpenHermes-2.5/prompt__cluster-00000-of-00001.parquet

@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:6146f35a5bb2563f68653737234af85b6d08ee262ccd9fcd05e2611011d45ead
-size 63047369