Update requirements.txt

requirements.txt

@@ -1,137 +1,61 @@
-accelerate==0.29.2
-aiofiles==23.2.1
-altair==5.3.0
-annotated-types==0.6.0
-anyio==4.3.0
-attrs==23.2.0
-av==11.0.0
-Brotli==1.1.0
-bs4
-certifi==2024.7.4
-charset-normalizer==3.3.2
-click==8.1.7
-colorama==0.4.6
-coloredlogs==15.0.1
-contourpy==1.2.1
-ctranslate2==4.2.1
-cycler==0.12.1
-dnspython==2.6.1
-email_validator==2.1.1
-fastapi==0.111.0
-fastapi-cli==0.0.3
-faster-whisper==1.0.1
-ffmpeg==1.4
-ffmpy==0.3.2
-filelock==3.13.4
-fire==0.6.0
-flatbuffers==24.3.25
-fonttools==4.51.0
-fsspec==2024.3.1
-gradio==4.29.0
-gradio_client==0.16.1
-h11==0.14.0
-httpcore==1.0.5
-httptools==0.6.1
-httpx==0.27.0
-huggingface-hub==0.22.2
-humanfriendly==10.0
-idna==3.7
-importlib_resources==6.4.0
-Jinja2==3.1.4
-jsonschema==4.22.0
-jsonschema-specifications==2023.12.1
-kiwisolver==1.4.5
-markdown-it-py==3.0.0
-MarkupSafe==2.1.5
-matplotlib==3.8.4
-mdurl==0.1.2
-mpmath==1.3.0
-mutagen==1.47.0
-networkx==3.3
-numpy==1.26.4
-onnxruntime==1.17.3
+arxiv
+beautifulsoup4
+bert_score
+chardet
+chromadb
+docx2txt
+EbookLib
+elasticsearch
+fastapi
+faster_whisper
+fire
+FlashRank
+fugashi
+gradio
+jieba
+Jinja2
+joblib
+langchain
+langdetect
+mwparserfromhell
+mwxml
+nltk
+numpy
+onnxruntime
 openai
-orjson==3.10.3
-packaging==24.0
-pandas==2.2.2
-pillow==10.3.0
+pandas
+Pillow
 playwright
-protobuf==5.26.1
-psutil==5.9.8
+psycopg2
 pyannote.audio
-pycryptodomex==3.20.0
-pydantic==2.7.1
-pydantic_core==2.18.2
-pydub==0.25.1
-pyee==11.1.0
-Pygments==2.18.0
-pypandoc_binary
-pypandoc
-pyparsing==3.1.2
-pyreadline3==3.4.1
-python-dateutil==2.9.0.post0
-python-dotenv==1.0.1
-python-ffmpeg==2.0.12
-python-multipart==0.0.9
-pytz==2024.1
-PyYAML==6.0.1
-referencing==0.35.1
-regex==2023.12.25
-requests==2.32.2
-rich==13.7.1
-rpds-py==0.18.1
-ruff==0.4.3
-safetensors==0.4.2
-semantic-version==2.10.0
-sentencepiece==0.2.0
-setuptools==69.5.1
-shellingham==1.5.4
-six==1.16.0
-scikit-learn
-sniffio==1.3.1
-starlette==0.37.2
-sympy==1.12
-termcolor==2.4.0
-timm==0.9.16
-tokenizers==0.15.2
-tomlkit==0.12.0
-toolz==0.12.1
-torchvision==0.17.2
-tqdm==4.66.3
-trafilatura
-transformers==4.39.3
-typer==0.12.3
-typing_extensions==4.11.0
-tzdata==2024.1
-ujson==5.9.0
-urllib3==2.2.2
-uvicorn==0.29.0
-watchfiles==0.21.0
-websockets==11.0.3
-yt-dlp
-nltk
-torch
-torchaudio
-openai~=1.28.1
-tiktoken~=0.6.0
+PyAudio
+pydub
 pymupdf
-chromadb
-protobuf==5.26.1
+pypandoc
+pypandoc_binary
+pytest
+python-dotenv
+PyYAML
+Requests
+rouge_score
+sacrebleu
+scikit_learn
+sentence_transformers
+SQLAlchemy
+streamlit
+tenacity
 textstat
-elasticsearch
-mwparserfromhell
-lxml_html_clean
-mwxml
-langdetect
-ebooklib
-docx2txt
+tiktoken
 toml
-chardet
+tqdm
+trafilatura
+transformers
+urllib3
+yt_dlp
 datasets
-arxiv
-lxml_html_clean
-#torch==2.2.2+cu121
-#torchaudio==2.2.2+cu121
-#websockets
-#yt-dlp==2024.4.9
-#--extra-index-url https://download.pytorch.org/whl/cu113
+tqdm
+--index-url https://download.pytorch.org/whl/cu124
+--extra-index-url https://pypi.org/simple
+torch
+torchaudio
+torchvision