Spaces:
Running
Running
| # for generate (gradio server) and finetune | |
| datasets==2.13.0 | |
| sentencepiece==0.1.99 | |
| gradio==3.41.2 | |
| huggingface_hub==0.16.4 | |
| appdirs==1.4.4 | |
| fire==0.5.0 | |
| docutils==0.20.1 | |
| torch==2.0.1; sys_platform != "darwin" and platform_machine != "arm64" | |
| evaluate==0.4.0 | |
| rouge_score==0.1.2 | |
| sacrebleu==2.3.1 | |
| scikit-learn==1.2.2 | |
| # optional (need to uncomment code in gradio_runner.py for import of better_profanity) | |
| # alt-profanity-check==1.2.2 | |
| # better-profanity==0.7.0 | |
| numpy==1.24.3 | |
| pandas==2.0.2 | |
| matplotlib==3.7.1 | |
| loralib==0.1.1 | |
| bitsandbytes==0.41.1 | |
| accelerate==0.22.0 | |
| peft==0.5.0 | |
| transformers==4.33.1 | |
| tokenizers==0.13.3 | |
| APScheduler==3.10.1 | |
| # optional for generate | |
| pynvml==11.5.0 | |
| psutil==5.9.5 | |
| boto3==1.26.101 | |
| botocore==1.29.101 | |
| # optional for finetune | |
| tensorboard==2.13.0 | |
| neptune==1.2.0 | |
| # for gradio client | |
| gradio_client==0.5.0 | |
| beautifulsoup4==4.12.2 | |
| markdown==3.4.3 | |
| # data and testing | |
| pytest==7.2.2 | |
| pytest-xdist==3.2.1 | |
| nltk==3.8.1 | |
| textstat==0.7.3 | |
| # pandoc==2.3 | |
| pypandoc==1.11; sys_platform == "darwin" and platform_machine == "arm64" | |
| pypandoc_binary==1.11; platform_machine == "x86_64" | |
| pypandoc_binary==1.11; sys_platform == "win32" | |
| python-magic-bin==0.4.14; sys_platform == "win32" | |
| openpyxl==3.1.2 | |
| lm_dataformat==0.0.20 | |
| bioc==2.0 | |
| # falcon | |
| einops==0.6.1 | |
| instructorembedding==1.0.1 | |
| # for gpt4all .env file, but avoid worrying about imports | |
| python-dotenv==1.0.0 | |
| text-generation==0.6.0 | |
| # for tokenization when don't have HF tokenizer | |
| tiktoken==0.4.0 | |
| requests>=2.31.0 | |
| urllib3>=1.26.16 | |
| filelock>=3.12.2 | |
| joblib>=1.3.1 | |
| tqdm>=4.65.0 | |
| tabulate>=0.9.0 | |
| packaging>=23.1 | |
| # optional for chat with PDF | |
| langchain==0.0.300 | |
| pypdf==3.14.0 | |
| # avoid textract, requires old six | |
| #textract==1.6.5 | |
| pypdfium2==4.19.0 | |
| # for HF embeddings | |
| sentence_transformers==2.2.2 | |
| # optional: for OpenAI endpoint or embeddings (requires key) | |
| openai==0.27.8 | |
| replicate==0.10.0 | |
| # local vector db | |
| chromadb==0.4.10 | |
| # chroma migration | |
| chroma-migrate==0.0.7 | |
| duckdb==0.7.1 | |
| https://h2o-release.s3.amazonaws.com/h2ogpt/chromamigdb-0.3.25-py3-none-any.whl | |
| https://h2o-release.s3.amazonaws.com/h2ogpt/hnswmiglib-0.7.0.tgz | |
| # server vector db | |
| #pymilvus==2.2.8 | |
| # weak url support, if can't install opencv etc. If comment-in this one, then comment-out unstructured[local-inference]==0.6.6 | |
| # unstructured==0.8.1 | |
| # strong support for images | |
| # Requires on Ubuntu: sudo apt-get install libmagic-dev poppler-utils tesseract-ocr libtesseract-dev libreoffice | |
| unstructured[local-inference]==0.9.0 | |
| #pdf2image==1.16.3 | |
| #pytesseract==0.3.10 | |
| pillow==9.5.0 | |
| posthog==3.0.1 | |
| pdfminer.six==20221105 | |
| urllib3 | |
| requests_file | |
| #pdf2image==1.16.3 | |
| #pytesseract==0.3.10 | |
| tabulate==0.9.0 | |
| # FYI pandoc already part of requirements.txt | |
| # JSONLoader, but makes some trouble for some users | |
| # TRY: apt-get install autoconf libtool | |
| # unclear what happens on windows/mac for now | |
| jq==1.4.1; platform_machine == "x86_64" | |
| # to check licenses | |
| # Run: pip-licenses|grep -v 'BSD\|Apache\|MIT' | |
| pip-licenses==4.3.0 | |
| # weaviate vector db | |
| weaviate-client==3.22.1 | |
| # optional for chat with PDF | |
| langchain==0.0.300 | |
| pypdf==3.14.0 | |
| # avoid textract, requires old six | |
| #textract==1.6.5 | |
| pypdfium2==4.19.0 | |
| # for HF embeddings | |
| sentence_transformers==2.2.2 | |
| # optional: for OpenAI endpoint or embeddings (requires key) | |
| openai==0.27.8 | |
| replicate==0.10.0 | |
| # local vector db | |
| chromadb==0.4.10 | |
| # chroma migration | |
| chroma-migrate==0.0.7 | |
| duckdb==0.7.1 | |
| https://h2o-release.s3.amazonaws.com/h2ogpt/chromamigdb-0.3.25-py3-none-any.whl | |
| https://h2o-release.s3.amazonaws.com/h2ogpt/hnswmiglib-0.7.0.tgz | |
| # server vector db | |
| #pymilvus==2.2.8 | |
| # weak url support, if can't install opencv etc. If comment-in this one, then comment-out unstructured[local-inference]==0.6.6 | |
| # unstructured==0.8.1 | |
| # strong support for images | |
| # Requires on Ubuntu: sudo apt-get install libmagic-dev poppler-utils tesseract-ocr libtesseract-dev libreoffice | |
| unstructured[local-inference]==0.9.0 | |
| #pdf2image==1.16.3 | |
| #pytesseract==0.3.10 | |
| pillow==9.5.0 | |
| posthog==3.0.1 | |
| pdfminer.six==20221105 | |
| urllib3 | |
| requests_file | |
| #pdf2image==1.16.3 | |
| #pytesseract==0.3.10 | |
| tabulate==0.9.0 | |
| # FYI pandoc already part of requirements.txt | |
| # JSONLoader, but makes some trouble for some users | |
| # TRY: apt-get install autoconf libtool | |
| # unclear what happens on windows/mac for now | |
| jq==1.4.1; platform_machine == "x86_64" | |
| # to check licenses | |
| # Run: pip-licenses|grep -v 'BSD\|Apache\|MIT' | |
| pip-licenses==4.3.0 | |
| # weaviate vector db | |
| weaviate-client==3.22.1 | |
| faiss-gpu==1.7.2 | |
| arxiv==1.4.8 | |
| pymupdf==1.23.1 # AGPL license | |
| # extract-msg==0.41.1 # GPL3 | |