Upload folder using huggingface_hub

.github/workflows/update_space.yml

@@ -0,0 +1,28 @@
+name: Run Python script
+
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v2
+
+    - name: Set up Python
+      uses: actions/setup-python@v2
+      with:
+        python-version: '3.9'
+
+    - name: Install Gradio
+      run: python -m pip install gradio
+
+    - name: Log in to Hugging Face
+      run: python -c 'import huggingface_hub; huggingface_hub.login(token="${{ secrets.hf_token }}")'
+
+    - name: Deploy to Spaces
+      run: gradio deploy

.gitignore

@@ -0,0 +1,162 @@
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# poetry
+#   Similar to Pipfile.lock, it is generally recommended to include poetry.lock in version control.
+#   This is especially recommended for binary packages to ensure reproducibility, and is more
+#   commonly ignored for libraries.
+#   https://python-poetry.org/docs/basic-usage/#commit-your-poetrylock-file-to-version-control
+#poetry.lock
+
+# pdm
+#   Similar to Pipfile.lock, it is generally recommended to include pdm.lock in version control.
+#pdm.lock
+#   pdm stores project-wide configurations in .pdm.toml, but it is recommended to not include it
+#   in version control.
+#   https://pdm.fming.dev/latest/usage/project/#working-with-version-control
+.pdm.toml
+.pdm-python
+.pdm-build/
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow and github.com/pdm-project/pdm
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+# PyCharm
+#  JetBrains specific template is maintained in a separate JetBrains.gitignore that can
+#  be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
+#  and can be added to the global gitignore or merged into this file.  For a more nuclear
+#  option (not recommended) you can uncomment the following to ignore the entire idea folder.
+#.idea/

README.md

@@ -1,12 +1,26 @@
----
-title: Demo Human Gpt
-emoji: 👀
-colorFrom: red
-colorTo: gray
-sdk: gradio
-sdk_version: 4.43.0
-app_file: app.py
-pinned: false
----
-
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+---
+title: demo_human_gpt
+app_file: interactive_test.py
+sdk: gradio
+sdk_version: 4.43.0
+---
+# HumanGPT Game Test
+<a target="_blank" href="https://colab.research.google.com/github/bldng1337/human_gpt_technical_test/blob/main/chat_test.ipynb">
+  <img src="https://colab.research.google.com/assets/colab-badge.svg" alt="Open In Colab"/>
+</a>
+
+This is a Test of the feasibility of letting an LLM generate the user part while the bot part is written by the user.
+
+We won't instruct the language model to roleplay as a user. Instead, we'll instruct it to generate the bot's responses as it was trained to do. Then, we let the model complete the user text blocks. Since the model doesn't distinguish between writing bot or user parts, we should be able to leverage its full training instead of trying to get it to rp which it was not trained for. Should also make gaslighting/confusing the model harder as its not pretending to be a user but should belive it is.
+
+## How to use
+Press the "Open in Colab" button to open the notebook in Google Colab.
+
+## TODO
+- Make a chatwindow with panel to test the model interactively
+- test multiple backs and forths
+
+## Models to compare/try:
+- https://huggingface.co/ArliAI/Llama-3.1-8B-ArliAI-RPMax-v1.1-GGUF
+- https://huggingface.co/ArliAI/Phi-3.5-mini-3.8B-ArliAI-RPMax-v1.1-GGUF
+- https://huggingface.co/google/gemma-2-2b-it-GGUF

chat_test.ipynb

@@ -0,0 +1,531 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "^C\n",
+      "Note: you may need to restart the kernel to use updated packages.\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "[notice] A new release of pip is available: 23.1.2 -> 24.2\n",
+      "[notice] To update, run: python.exe -m pip install --upgrade pip\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Collecting llama-cpp-python\n",
+      "  Downloading llama_cpp_python-0.2.90.tar.gz (63.8 MB)\n",
+      "                                              0.0/63.8 MB ? eta -:--:--\n",
+      "                                             0.0/63.8 MB 960.0 kB/s eta 0:01:07\n",
+      "                                              0.2/63.8 MB 2.1 MB/s eta 0:00:31\n",
+      "                                              0.4/63.8 MB 2.9 MB/s eta 0:00:23\n",
+      "                                              0.7/63.8 MB 3.8 MB/s eta 0:00:17\n",
+      "                                              1.1/63.8 MB 5.0 MB/s eta 0:00:13\n",
+      "                                              1.3/63.8 MB 4.9 MB/s eta 0:00:13\n",
+      "     -                                        1.8/63.8 MB 5.6 MB/s eta 0:00:12\n",
+      "     -                                        1.9/63.8 MB 5.1 MB/s eta 0:00:13\n",
+      "     -                                        2.1/63.8 MB 5.2 MB/s eta 0:00:12\n",
+      "     -                                        2.6/63.8 MB 5.8 MB/s eta 0:00:11\n",
+      "     -                                        3.0/63.8 MB 5.9 MB/s eta 0:00:11\n",
+      "     --                                       3.5/63.8 MB 6.3 MB/s eta 0:00:10\n",
+      "     --                                       3.9/63.8 MB 6.6 MB/s eta 0:00:10\n",
+      "     --                                       4.5/63.8 MB 7.0 MB/s eta 0:00:09\n",
+      "     ---                                      4.9/63.8 MB 7.1 MB/s eta 0:00:09\n",
+      "     ---                                      5.1/63.8 MB 7.1 MB/s eta 0:00:09\n",
+      "     ---                                      5.5/63.8 MB 7.0 MB/s eta 0:00:09\n",
+      "     ---                                      5.9/63.8 MB 7.1 MB/s eta 0:00:09\n",
+      "     ----                                     6.4/63.8 MB 7.3 MB/s eta 0:00:08\n",
+      "     ----                                     6.7/63.8 MB 7.3 MB/s eta 0:00:08\n",
+      "     ----                                     7.2/63.8 MB 7.4 MB/s eta 0:00:08\n",
+      "     ----                                     7.5/63.8 MB 7.4 MB/s eta 0:00:08\n",
+      "     -----                                    8.0/63.8 MB 7.6 MB/s eta 0:00:08\n",
+      "     -----                                    8.6/63.8 MB 7.7 MB/s eta 0:00:08\n",
+      "     -----                                    9.1/63.8 MB 7.8 MB/s eta 0:00:07\n",
+      "     ------                                   9.6/63.8 MB 8.0 MB/s eta 0:00:07\n",
+      "     ------                                   9.9/63.8 MB 7.9 MB/s eta 0:00:07\n",
+      "     ------                                   10.6/63.8 MB 8.7 MB/s eta 0:00:07\n",
+      "     ------                                   11.1/63.8 MB 9.0 MB/s eta 0:00:06\n",
+      "     -------                                  11.5/63.8 MB 9.1 MB/s eta 0:00:06\n",
+      "     -------                                  11.9/63.8 MB 9.1 MB/s eta 0:00:06\n",
+      "     -------                                  12.6/63.8 MB 9.8 MB/s eta 0:00:06\n",
+      "     --------                                 13.1/63.8 MB 9.8 MB/s eta 0:00:06\n",
+      "     --------                                 13.5/63.8 MB 9.8 MB/s eta 0:00:06\n",
+      "     --------                                 13.9/63.8 MB 9.8 MB/s eta 0:00:06\n",
+      "     --------                                 14.3/63.8 MB 9.8 MB/s eta 0:00:06\n",
+      "     ---------                                14.8/63.8 MB 9.6 MB/s eta 0:00:06\n",
+      "     ---------                               15.4/63.8 MB 10.2 MB/s eta 0:00:05\n",
+      "     ---------                               16.1/63.8 MB 10.4 MB/s eta 0:00:05\n",
+      "     ----------                              16.6/63.8 MB 10.4 MB/s eta 0:00:05\n",
+      "     ----------                               16.6/63.8 MB 9.9 MB/s eta 0:00:05\n",
+      "     ----------                              17.0/63.8 MB 10.1 MB/s eta 0:00:05\n",
+      "     ----------                              17.7/63.8 MB 10.4 MB/s eta 0:00:05\n",
+      "     -----------                             18.4/63.8 MB 10.6 MB/s eta 0:00:05\n",
+      "     -----------                             18.9/63.8 MB 10.6 MB/s eta 0:00:05\n",
+      "     -----------                             19.4/63.8 MB 10.6 MB/s eta 0:00:05\n",
+      "     ------------                            19.9/63.8 MB 10.7 MB/s eta 0:00:05\n",
+      "     ------------                            20.3/63.8 MB 10.7 MB/s eta 0:00:05\n",
+      "     ------------                            20.8/63.8 MB 10.4 MB/s eta 0:00:05\n",
+      "     -------------                           21.4/63.8 MB 10.6 MB/s eta 0:00:05\n",
+      "     -------------                           21.9/63.8 MB 10.7 MB/s eta 0:00:04\n",
+      "     -------------                           22.5/63.8 MB 10.7 MB/s eta 0:00:04\n",
+      "     -------------                           22.9/63.8 MB 10.6 MB/s eta 0:00:04\n",
+      "     --------------                          23.5/63.8 MB 10.6 MB/s eta 0:00:04\n",
+      "     --------------                          23.7/63.8 MB 10.4 MB/s eta 0:00:04\n",
+      "     --------------                          24.1/63.8 MB 10.6 MB/s eta 0:00:04\n",
+      "     --------------                          24.4/63.8 MB 10.4 MB/s eta 0:00:04\n",
+      "     ---------------                         24.8/63.8 MB 10.2 MB/s eta 0:00:04\n",
+      "     ---------------                         25.2/63.8 MB 10.2 MB/s eta 0:00:04\n",
+      "     ----------------                         25.7/63.8 MB 9.9 MB/s eta 0:00:04\n",
+      "     ----------------                         26.2/63.8 MB 9.9 MB/s eta 0:00:04\n",
+      "     ----------------                         26.6/63.8 MB 9.8 MB/s eta 0:00:04\n",
+      "     ----------------                        26.9/63.8 MB 10.1 MB/s eta 0:00:04\n",
+      "     ----------------                        27.3/63.8 MB 10.1 MB/s eta 0:00:04\n",
+      "     -----------------                        27.8/63.8 MB 9.9 MB/s eta 0:00:04\n",
+      "     -----------------                        28.3/63.8 MB 9.8 MB/s eta 0:00:04\n",
+      "     -----------------                        28.7/63.8 MB 9.6 MB/s eta 0:00:04\n",
+      "     ------------------                       29.0/63.8 MB 9.4 MB/s eta 0:00:04\n",
+      "     ------------------                       29.2/63.8 MB 9.2 MB/s eta 0:00:04\n",
+      "     ------------------                       29.7/63.8 MB 9.4 MB/s eta 0:00:04\n",
+      "     ------------------                       30.1/63.8 MB 9.1 MB/s eta 0:00:04\n",
+      "     -------------------                      30.3/63.8 MB 8.8 MB/s eta 0:00:04\n",
+      "     -------------------                      30.8/63.8 MB 9.0 MB/s eta 0:00:04\n",
+      "     -------------------                      31.0/63.8 MB 8.7 MB/s eta 0:00:04\n",
+      "     -------------------                      31.3/63.8 MB 8.5 MB/s eta 0:00:04\n",
+      "     -------------------                      31.6/63.8 MB 8.4 MB/s eta 0:00:04\n",
+      "     --------------------                     32.0/63.8 MB 8.4 MB/s eta 0:00:04\n",
+      "     --------------------                     32.4/63.8 MB 8.3 MB/s eta 0:00:04\n",
+      "     --------------------                     32.9/63.8 MB 8.3 MB/s eta 0:00:04\n",
+      "     --------------------                     33.3/63.8 MB 8.3 MB/s eta 0:00:04\n",
+      "     ---------------------                    33.7/63.8 MB 8.1 MB/s eta 0:00:04\n",
+      "     ---------------------                    34.2/63.8 MB 8.3 MB/s eta 0:00:04\n",
+      "     ---------------------                    34.7/63.8 MB 8.5 MB/s eta 0:00:04\n",
+      "     ----------------------                   35.1/63.8 MB 8.5 MB/s eta 0:00:04\n",
+      "     ----------------------                   35.5/63.8 MB 8.5 MB/s eta 0:00:04\n",
+      "     ----------------------                   36.0/63.8 MB 8.4 MB/s eta 0:00:04\n",
+      "     ----------------------                   36.3/63.8 MB 8.3 MB/s eta 0:00:04\n",
+      "     -----------------------                  36.8/63.8 MB 8.4 MB/s eta 0:00:04\n",
+      "     -----------------------                  37.2/63.8 MB 8.5 MB/s eta 0:00:04\n",
+      "     -----------------------                  37.6/63.8 MB 8.5 MB/s eta 0:00:04\n",
+      "     -----------------------                  38.1/63.8 MB 8.4 MB/s eta 0:00:04\n",
+      "     ------------------------                 38.6/63.8 MB 8.6 MB/s eta 0:00:03\n",
+      "     ------------------------                 39.2/63.8 MB 8.7 MB/s eta 0:00:03\n",
+      "     ------------------------                 39.6/63.8 MB 8.8 MB/s eta 0:00:03\n",
+      "     -------------------------                40.1/63.8 MB 9.0 MB/s eta 0:00:03\n",
+      "     -------------------------                40.5/63.8 MB 9.2 MB/s eta 0:00:03\n",
+      "     -------------------------                41.0/63.8 MB 9.2 MB/s eta 0:00:03\n",
+      "     -------------------------                41.4/63.8 MB 9.4 MB/s eta 0:00:03\n",
+      "     --------------------------               41.7/63.8 MB 9.2 MB/s eta 0:00:03\n",
+      "     --------------------------               42.3/63.8 MB 9.6 MB/s eta 0:00:03\n",
+      "     --------------------------               42.8/63.8 MB 9.6 MB/s eta 0:00:03\n",
+      "     ---------------------------              43.3/63.8 MB 9.6 MB/s eta 0:00:03\n",
+      "     ---------------------------              43.6/63.8 MB 9.6 MB/s eta 0:00:03\n",
+      "     ---------------------------              44.2/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     ----------------------------             44.7/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     ----------------------------             45.2/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     ----------------------------             45.6/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     ----------------------------             46.1/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     -----------------------------            46.5/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     -----------------------------            46.9/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     -----------------------------            47.4/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     -----------------------------           47.9/63.8 MB 10.1 MB/s eta 0:00:02\n",
+      "     -----------------------------           48.4/63.8 MB 10.1 MB/s eta 0:00:02\n",
+      "     ------------------------------           48.8/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     ------------------------------           49.4/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     -------------------------------          49.8/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     -------------------------------          50.2/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     -------------------------------          50.8/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     --------------------------------         51.2/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     --------------------------------         51.5/63.8 MB 9.8 MB/s eta 0:00:02\n",
+      "     --------------------------------         52.0/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     --------------------------------         52.4/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     ---------------------------------        52.9/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     ---------------------------------        53.4/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     ---------------------------------        53.8/63.8 MB 9.9 MB/s eta 0:00:02\n",
+      "     ----------------------------------       54.4/63.8 MB 9.9 MB/s eta 0:00:01\n",
+      "     ----------------------------------       54.7/63.8 MB 9.6 MB/s eta 0:00:01\n",
+      "     ----------------------------------       55.2/63.8 MB 9.8 MB/s eta 0:00:01\n",
+      "     ----------------------------------       55.5/63.8 MB 9.6 MB/s eta 0:00:01\n",
+      "     -----------------------------------      56.0/63.8 MB 9.6 MB/s eta 0:00:01\n",
+      "     -----------------------------------      56.4/63.8 MB 9.8 MB/s eta 0:00:01\n",
+      "     -----------------------------------      57.1/63.8 MB 9.9 MB/s eta 0:00:01\n",
+      "     -----------------------------------     57.7/63.8 MB 10.1 MB/s eta 0:00:01\n",
+      "     ------------------------------------     58.1/63.8 MB 9.9 MB/s eta 0:00:01\n",
+      "     ------------------------------------     58.6/63.8 MB 9.9 MB/s eta 0:00:01\n",
+      "     ------------------------------------    59.1/63.8 MB 10.1 MB/s eta 0:00:01\n",
+      "     ------------------------------------    59.7/63.8 MB 10.1 MB/s eta 0:00:01\n",
+      "     ------------------------------------    60.2/63.8 MB 10.1 MB/s eta 0:00:01\n",
+      "     -------------------------------------   60.8/63.8 MB 10.2 MB/s eta 0:00:01\n",
+      "     -------------------------------------   61.3/63.8 MB 10.2 MB/s eta 0:00:01\n",
+      "     -------------------------------------   61.9/63.8 MB 10.6 MB/s eta 0:00:01\n",
+      "     -------------------------------------   61.9/63.8 MB 10.6 MB/s eta 0:00:01\n",
+      "     --------------------------------------  62.3/63.8 MB 10.1 MB/s eta 0:00:01\n",
+      "     --------------------------------------  62.9/63.8 MB 10.2 MB/s eta 0:00:01\n",
+      "     --------------------------------------  63.2/63.8 MB 10.1 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  63.8/63.8 MB 9.9 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  63.8/63.8 MB 9.9 MB/s eta 0:00:01\n",
+      "     ---------------------------------------  63.8/63.8 MB 9.9 MB/s eta 0:00:01\n",
+      "     ---------------------------------------- 63.8/63.8 MB 8.8 MB/s eta 0:00:00\n",
+      "  Installing build dependencies: started\n",
+      "  Installing build dependencies: finished with status 'done'\n",
+      "  Getting requirements to build wheel: started\n",
+      "  Getting requirements to build wheel: finished with status 'done'\n",
+      "  Preparing metadata (pyproject.toml): started\n",
+      "  Preparing metadata (pyproject.toml): finished with status 'done'\n",
+      "Requirement already satisfied: typing-extensions>=4.5.0 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from llama-cpp-python) (4.12.2)\n",
+      "Requirement already satisfied: numpy>=1.20.0 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from llama-cpp-python) (2.1.1)\n",
+      "Collecting diskcache>=5.6.1 (from llama-cpp-python)\n",
+      "  Downloading diskcache-5.6.3-py3-none-any.whl (45 kB)\n",
+      "                                              0.0/45.5 kB ? eta -:--:--\n",
+      "     ---------------------------------------- 45.5/45.5 kB ? eta 0:00:00\n",
+      "Requirement already satisfied: jinja2>=2.11.3 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from llama-cpp-python) (3.1.4)\n",
+      "Requirement already satisfied: MarkupSafe>=2.0 in c:\\users\\bldng\\dev\\demo_human_gpt\\.venv\\lib\\site-packages (from jinja2>=2.11.3->llama-cpp-python) (2.1.5)\n",
+      "Building wheels for collected packages: llama-cpp-python\n",
+      "  Building wheel for llama-cpp-python (pyproject.toml): started\n",
+      "  Building wheel for llama-cpp-python (pyproject.toml): still running...\n",
+      "  Building wheel for llama-cpp-python (pyproject.toml): finished with status 'done'\n",
+      "  Created wheel for llama-cpp-python: filename=llama_cpp_python-0.2.90-cp311-cp311-win_amd64.whl size=3132437 sha256=b3123f77abab7606d9e29442b91118b19ce2419f505ffe71f599da6060482312\n",
+      "  Stored in directory: c:\\users\\bldng\\appdata\\local\\pip\\cache\\wheels\\f8\\bd\\3d\\dcb25387572a6fe6c9c2953597c4c4d6dd985a425475da444d\n",
+      "Successfully built llama-cpp-python\n",
+      "Installing collected packages: diskcache, llama-cpp-python\n",
+      "Successfully installed diskcache-5.6.3 llama-cpp-python-0.2.90\n"
+     ]
+    }
+   ],
+   "source": [
+    "%pip install llama-cpp-python"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 27,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "llama_model_loader: loaded meta data with 40 key-value pairs and 197 tensors from C:\\Users\\bldng\\.cache\\huggingface\\hub\\models--bartowski--Phi-3.5-mini-instruct-GGUF\\snapshots\\5c8d5381f90b6ca4348f090238be34eada23d192\\.\\Phi-3.5-mini-instruct-IQ3_XS.gguf (version GGUF V3 (latest))\n",
+      "llama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\n",
+      "llama_model_loader: - kv   0:                       general.architecture str              = phi3\n",
+      "llama_model_loader: - kv   1:                               general.type str              = model\n",
+      "llama_model_loader: - kv   2:                               general.name str              = Phi 3.5 Mini Instruct\n",
+      "llama_model_loader: - kv   3:                           general.finetune str              = instruct\n",
+      "llama_model_loader: - kv   4:                           general.basename str              = Phi-3.5\n",
+      "llama_model_loader: - kv   5:                         general.size_label str              = mini\n",
+      "llama_model_loader: - kv   6:                            general.license str              = mit\n",
+      "llama_model_loader: - kv   7:                       general.license.link str              = https://huggingface.co/microsoft/Phi-...\n",
+      "llama_model_loader: - kv   8:                               general.tags arr[str,3]       = [\"nlp\", \"code\", \"text-generation\"]\n",
+      "llama_model_loader: - kv   9:                          general.languages arr[str,1]       = [\"multilingual\"]\n",
+      "llama_model_loader: - kv  10:                        phi3.context_length u32              = 131072\n",
+      "llama_model_loader: - kv  11:  phi3.rope.scaling.original_context_length u32              = 4096\n",
+      "llama_model_loader: - kv  12:                      phi3.embedding_length u32              = 3072\n",
+      "llama_model_loader: - kv  13:                   phi3.feed_forward_length u32              = 8192\n",
+      "llama_model_loader: - kv  14:                           phi3.block_count u32              = 32\n",
+      "llama_model_loader: - kv  15:                  phi3.attention.head_count u32              = 32\n",
+      "llama_model_loader: - kv  16:               phi3.attention.head_count_kv u32              = 32\n",
+      "llama_model_loader: - kv  17:      phi3.attention.layer_norm_rms_epsilon f32              = 0.000010\n",
+      "llama_model_loader: - kv  18:                  phi3.rope.dimension_count u32              = 96\n",
+      "llama_model_loader: - kv  19:                        phi3.rope.freq_base f32              = 10000.000000\n",
+      "llama_model_loader: - kv  20:                          general.file_type u32              = 22\n",
+      "llama_model_loader: - kv  21:              phi3.attention.sliding_window u32              = 262144\n",
+      "llama_model_loader: - kv  22:              phi3.rope.scaling.attn_factor f32              = 1.190238\n",
+      "llama_model_loader: - kv  23:                       tokenizer.ggml.model str              = llama\n",
+      "llama_model_loader: - kv  24:                         tokenizer.ggml.pre str              = default\n",
+      "llama_model_loader: - kv  25:                      tokenizer.ggml.tokens arr[str,32064]   = [\"<unk>\", \"<s>\", \"</s>\", \"<0x00>\", \"<...\n",
+      "llama_model_loader: - kv  26:                      tokenizer.ggml.scores arr[f32,32064]   = [-1000.000000, -1000.000000, -1000.00...\n",
+      "llama_model_loader: - kv  27:                  tokenizer.ggml.token_type arr[i32,32064]   = [3, 3, 4, 6, 6, 6, 6, 6, 6, 6, 6, 6, ...\n",
+      "llama_model_loader: - kv  28:                tokenizer.ggml.bos_token_id u32              = 1\n",
+      "llama_model_loader: - kv  29:                tokenizer.ggml.eos_token_id u32              = 32000\n",
+      "llama_model_loader: - kv  30:            tokenizer.ggml.unknown_token_id u32              = 0\n",
+      "llama_model_loader: - kv  31:            tokenizer.ggml.padding_token_id u32              = 32000\n",
+      "llama_model_loader: - kv  32:               tokenizer.ggml.add_bos_token bool             = false\n",
+      "llama_model_loader: - kv  33:               tokenizer.ggml.add_eos_token bool             = false\n",
+      "llama_model_loader: - kv  34:                    tokenizer.chat_template str              = {% for message in messages %}{% if me...\n",
+      "llama_model_loader: - kv  35:               general.quantization_version u32              = 2\n",
+      "llama_model_loader: - kv  36:                      quantize.imatrix.file str              = /models_out/Phi-3.5-mini-instruct-GGU...\n",
+      "llama_model_loader: - kv  37:                   quantize.imatrix.dataset str              = /training_dir/calibration_datav3.txt\n",
+      "llama_model_loader: - kv  38:             quantize.imatrix.entries_count i32              = 128\n",
+      "llama_model_loader: - kv  39:              quantize.imatrix.chunks_count i32              = 151\n",
+      "llama_model_loader: - type  f32:   67 tensors\n",
+      "llama_model_loader: - type q6_K:    1 tensors\n",
+      "llama_model_loader: - type iq3_xxs:   24 tensors\n",
+      "llama_model_loader: - type iq3_s:  105 tensors\n",
+      "llm_load_vocab: special tokens cache size = 14\n",
+      "llm_load_vocab: token to piece cache size = 0.1685 MB\n",
+      "llm_load_print_meta: format           = GGUF V3 (latest)\n",
+      "llm_load_print_meta: arch             = phi3\n",
+      "llm_load_print_meta: vocab type       = SPM\n",
+      "llm_load_print_meta: n_vocab          = 32064\n",
+      "llm_load_print_meta: n_merges         = 0\n",
+      "llm_load_print_meta: vocab_only       = 0\n",
+      "llm_load_print_meta: n_ctx_train      = 131072\n",
+      "llm_load_print_meta: n_embd           = 3072\n",
+      "llm_load_print_meta: n_layer          = 32\n",
+      "llm_load_print_meta: n_head           = 32\n",
+      "llm_load_print_meta: n_head_kv        = 32\n",
+      "llm_load_print_meta: n_rot            = 96\n",
+      "llm_load_print_meta: n_swa            = 262144\n",
+      "llm_load_print_meta: n_embd_head_k    = 96\n",
+      "llm_load_print_meta: n_embd_head_v    = 96\n",
+      "llm_load_print_meta: n_gqa            = 1\n",
+      "llm_load_print_meta: n_embd_k_gqa     = 3072\n",
+      "llm_load_print_meta: n_embd_v_gqa     = 3072\n",
+      "llm_load_print_meta: f_norm_eps       = 0.0e+00\n",
+      "llm_load_print_meta: f_norm_rms_eps   = 1.0e-05\n",
+      "llm_load_print_meta: f_clamp_kqv      = 0.0e+00\n",
+      "llm_load_print_meta: f_max_alibi_bias = 0.0e+00\n",
+      "llm_load_print_meta: f_logit_scale    = 0.0e+00\n",
+      "llm_load_print_meta: n_ff             = 8192\n",
+      "llm_load_print_meta: n_expert         = 0\n",
+      "llm_load_print_meta: n_expert_used    = 0\n",
+      "llm_load_print_meta: causal attn      = 1\n",
+      "llm_load_print_meta: pooling type     = 0\n",
+      "llm_load_print_meta: rope type        = 2\n",
+      "llm_load_print_meta: rope scaling     = linear\n",
+      "llm_load_print_meta: freq_base_train  = 10000.0\n",
+      "llm_load_print_meta: freq_scale_train = 1\n",
+      "llm_load_print_meta: n_ctx_orig_yarn  = 4096\n",
+      "llm_load_print_meta: rope_finetuned   = unknown\n",
+      "llm_load_print_meta: ssm_d_conv       = 0\n",
+      "llm_load_print_meta: ssm_d_inner      = 0\n",
+      "llm_load_print_meta: ssm_d_state      = 0\n",
+      "llm_load_print_meta: ssm_dt_rank      = 0\n",
+      "llm_load_print_meta: ssm_dt_b_c_rms   = 0\n",
+      "llm_load_print_meta: model type       = 3B\n",
+      "llm_load_print_meta: model ftype      = IQ3_XS - 3.3 bpw\n",
+      "llm_load_print_meta: model params     = 3.82 B\n",
+      "llm_load_print_meta: model size       = 1.51 GiB (3.40 BPW) \n",
+      "llm_load_print_meta: general.name     = Phi 3.5 Mini Instruct\n",
+      "llm_load_print_meta: BOS token        = 1 '<s>'\n",
+      "llm_load_print_meta: EOS token        = 32000 '<|endoftext|>'\n",
+      "llm_load_print_meta: UNK token        = 0 '<unk>'\n",
+      "llm_load_print_meta: PAD token        = 32000 '<|endoftext|>'\n",
+      "llm_load_print_meta: LF token         = 13 '<0x0A>'\n",
+      "llm_load_print_meta: EOT token        = 32007 '<|end|>'\n",
+      "llm_load_print_meta: max token length = 48\n",
+      "llm_load_tensors: ggml ctx size =    0.10 MiB\n",
+      "llm_load_tensors:        CPU buffer size =  1549.18 MiB\n",
+      ".......................................................................................\n",
+      "llama_new_context_with_model: n_ctx      = 512\n",
+      "llama_new_context_with_model: n_batch    = 512\n",
+      "llama_new_context_with_model: n_ubatch   = 512\n",
+      "llama_new_context_with_model: flash_attn = 0\n",
+      "llama_new_context_with_model: freq_base  = 10000.0\n",
+      "llama_new_context_with_model: freq_scale = 1\n",
+      "llama_kv_cache_init:        CPU KV buffer size =   192.00 MiB\n",
+      "llama_new_context_with_model: KV self size  =  192.00 MiB, K (f16):   96.00 MiB, V (f16):   96.00 MiB\n",
+      "llama_new_context_with_model:        CPU  output buffer size =     0.12 MiB\n",
+      "llama_new_context_with_model:        CPU compute buffer size =    83.01 MiB\n",
+      "llama_new_context_with_model: graph nodes  = 1286\n",
+      "llama_new_context_with_model: graph splits = 1\n",
+      "AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 0 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | AVX512_BF16 = 0 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | WASM_SIMD = 0 | BLAS = 0 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \n",
+      "Model metadata: {'general.name': 'Phi 3.5 Mini Instruct', 'general.architecture': 'phi3', 'general.type': 'model', 'general.basename': 'Phi-3.5', 'general.finetune': 'instruct', 'general.size_label': 'mini', 'general.license': 'mit', 'general.license.link': 'https://huggingface.co/microsoft/Phi-3.5-mini-instruct/resolve/main/LICENSE', 'phi3.attention.head_count_kv': '32', 'phi3.context_length': '131072', 'phi3.rope.scaling.original_context_length': '4096', 'phi3.embedding_length': '3072', 'tokenizer.ggml.model': 'llama', 'phi3.feed_forward_length': '8192', 'phi3.block_count': '32', 'phi3.attention.head_count': '32', 'phi3.attention.layer_norm_rms_epsilon': '0.000010', 'phi3.rope.dimension_count': '96', 'tokenizer.chat_template': \"{% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\\n' + message['content'] + '<|end|>\\n'}}{% elif message['role'] == 'user' %}{{'<|user|>\\n' + message['content'] + '<|end|>\\n'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\\n' + message['content'] + '<|end|>\\n'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\\n' }}{% else %}{{ eos_token }}{% endif %}\", 'phi3.rope.freq_base': '10000.000000', 'tokenizer.ggml.eos_token_id': '32000', 'general.file_type': '22', 'tokenizer.ggml.add_eos_token': 'false', 'phi3.attention.sliding_window': '262144', 'phi3.rope.scaling.attn_factor': '1.190238', 'tokenizer.ggml.pre': 'default', 'general.quantization_version': '2', 'tokenizer.ggml.bos_token_id': '1', 'tokenizer.ggml.unknown_token_id': '0', 'tokenizer.ggml.padding_token_id': '32000', 'tokenizer.ggml.add_bos_token': 'false', 'quantize.imatrix.chunks_count': '151', 'quantize.imatrix.file': '/models_out/Phi-3.5-mini-instruct-GGUF/Phi-3.5-mini-instruct.imatrix', 'quantize.imatrix.dataset': '/training_dir/calibration_datav3.txt', 'quantize.imatrix.entries_count': '128'}\n",
+      "Available chat formats from metadata: chat_template.default\n",
+      "Using gguf chat template: {% for message in messages %}{% if message['role'] == 'system' and message['content'] %}{{'<|system|>\n",
+      "' + message['content'] + '<|end|>\n",
+      "'}}{% elif message['role'] == 'user' %}{{'<|user|>\n",
+      "' + message['content'] + '<|end|>\n",
+      "'}}{% elif message['role'] == 'assistant' %}{{'<|assistant|>\n",
+      "' + message['content'] + '<|end|>\n",
+      "'}}{% endif %}{% endfor %}{% if add_generation_prompt %}{{ '<|assistant|>\n",
+      "' }}{% else %}{{ eos_token }}{% endif %}\n",
+      "Using chat eos_token: <|endoftext|>\n",
+      "Using chat bos_token: <s>\n"
+     ]
+    }
+   ],
+   "source": [
+    "from llama_cpp import Llama\n",
+    "\n",
+    "llm = Llama.from_pretrained(\n",
+    "\trepo_id=\"bartowski/Phi-3.5-mini-instruct-GGUF\",\n",
+    "\tfilename=\"Phi-3.5-mini-instruct-IQ3_XS.gguf\",\n",
+    ")\n",
+    "\n"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 41,
+   "metadata": {},
+   "outputs": [
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "Test 1\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Llama.generate: 195 prefix-match hit, remaining 22 prompt tokens to eval\n",
+      "\n",
+      "llama_print_timings:        load time =    1880.42 ms\n",
+      "llama_print_timings:      sample time =       0.27 ms /     9 runs   (    0.03 ms per token, 33333.33 tokens per second)\n",
+      "llama_print_timings: prompt eval time =    1371.67 ms /    22 tokens (   62.35 ms per token,    16.04 tokens per second)\n",
+      "llama_print_timings:        eval time =     791.79 ms /     8 runs   (   98.97 ms per token,    10.10 tokens per second)\n",
+      "llama_print_timings:       total time =    2166.71 ms /    30 tokens\n",
+      "Llama.generate: 196 prefix-match hit, remaining 26 prompt tokens to eval\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<|user|>\n",
+      "What is the capital of France?<|end|>\n",
+      "<|assistant|>\n",
+      "The capital of France is Paris<|end|>\n",
+      "<|user|>\n",
+      " \n",
+      "<|end\n",
+      "Test 2\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "llama_print_timings:        load time =    1880.42 ms\n",
+      "llama_print_timings:      sample time =       0.30 ms /     9 runs   (    0.03 ms per token, 29801.32 tokens per second)\n",
+      "llama_print_timings: prompt eval time =    1561.16 ms /    26 tokens (   60.04 ms per token,    16.65 tokens per second)\n",
+      "llama_print_timings:        eval time =     787.52 ms /     8 runs   (   98.44 ms per token,    10.16 tokens per second)\n",
+      "llama_print_timings:       total time =    2352.22 ms /    34 tokens\n",
+      "Llama.generate: 195 prefix-match hit, remaining 23 prompt tokens to eval\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<|user|>\n",
+      "What is 3*4?<|end|>\n",
+      "<|assistant|>\n",
+      "The result of 3*4 is 12<|end|>\n",
+      "<|user|>\n",
+      " \n",
+      "<|end\n",
+      "Test 3\n"
+     ]
+    },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\n",
+      "llama_print_timings:        load time =    1880.42 ms\n",
+      "llama_print_timings:      sample time =       0.21 ms /     7 runs   (    0.03 ms per token, 32863.85 tokens per second)\n",
+      "llama_print_timings: prompt eval time =    1388.34 ms /    23 tokens (   60.36 ms per token,    16.57 tokens per second)\n",
+      "llama_print_timings:        eval time =     578.12 ms /     6 runs   (   96.35 ms per token,    10.38 tokens per second)\n",
+      "llama_print_timings:       total time =    1969.21 ms /    29 tokens\n"
+     ]
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "<|user|>\n",
+      "What it the capital?<|end|>\n",
+      "<|assistant|>\n",
+      "Please specify which country you are referring to.<|end|>\n",
+      "<|user|>\n",
+      " \n",
+      "The capital of Italy\n"
+     ]
+    }
+   ],
+   "source": [
+    "syspropmt=r\"\"\"\n",
+    "<|system|>\n",
+    "The User will make an inquiry to the assistant.\n",
+    "Fullfill the users inquiry.\n",
+    "The User will write a message with the content \"<|endtile|>\" if his inquiry is fullfilled.\n",
+    "The User will never have more than one inquiry in one conversation.\n",
+    "The User will never complete his own inquiry.\n",
+    "All conversations will end with \"<|endtile|>\".\n",
+    "After each User message is one assistant response.\n",
+    "There can never be more than one assistant response in succession.\n",
+    "The end of each user and assistant message is marked with \"<|end|>\".\n",
+    "\n",
+    "\n",
+    "Example:\n",
+    "<|user|>\n",
+    "What is the capital?<|end|>\n",
+    "<|assistant|>\n",
+    "Could you please specify which capital you are referring to?<|end|>\n",
+    "<|user|>\n",
+    "The capital of France<|end|>\n",
+    "<|assistant|>\n",
+    "The capital of France is Paris\n",
+    "<|user|>\n",
+    "<|endtile|><|end|>\n",
+    "\"\"\".strip()\n",
+    "tests=[\n",
+    "\"\"\"\n",
+    "<|user|>\n",
+    "What is the capital of France?<|end|>\n",
+    "<|assistant|>\n",
+    "The capital of France is Paris<|end|>\n",
+    "<|user|>\n",
+    "\"\"\".strip(),#Should terminate the conversation with <|endtile|>\n",
+    "\"\"\"\n",
+    "<|user|>\n",
+    "What is 3*4?<|end|>\n",
+    "<|assistant|>\n",
+    "The result of 3*4 is 12<|end|>\n",
+    "<|user|>\n",
+    "\"\"\".strip(),#Should terminate the conversation with <|endtile|>\n",
+    "\"\"\"\n",
+    "<|user|>\n",
+    "What it the capital?<|end|>\n",
+    "<|assistant|>\n",
+    "Please specify which country you are referring to.<|end|>\n",
+    "<|user|>\n",
+    "\"\"\".strip(),#Should specify the country\n",
+    "]\n",
+    "for num,test in enumerate(tests):\n",
+    "    print(f\"Test {num+1}\")\n",
+    "    output=llm(syspropmt+test, stop=[\"\\n \",\"?\\n\",\".\\n\",\"tile|>\"],max_tokens=100)[\"choices\"][0][\"text\"]#Only stop at tile|> to see if the conv got terminated\n",
+    "    print(test)\n",
+    "    print(output)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": ".venv",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.11.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

interactive_test.py

@@ -0,0 +1,71 @@
+from re import sub
+from typing import Dict, List
+import gradio as gr
+from llama_cpp import Llama
+import random
+import time
+
+from sympy import Li
+
+syspropmt=r"""
+<|system|>
+The User will make an inquiry to the assistant.
+Fullfill the users inquiry.
+The User will write a message with the content "<|endtile|>" if his inquiry is fullfilled.
+The User will never have more than one inquiry in one conversation.
+The User will never complete his own inquiry.
+All conversations will end with "<|endtile|>".
+After each User message is one assistant response.
+There can never be more than one assistant response in succession.
+The end of each user and assistant message is marked with "<|end|>".
+
+
+Example:
+<|user|>
+What is the capital?<|end|>
+<|assistant|>
+Could you please specify which capital you are referring to?<|end|>
+<|user|>
+The capital of France<|end|>
+<|assistant|>
+The capital of France is Paris
+<|user|>
+<|endtile|><|end|>
+""".strip()
+
+llm = Llama.from_pretrained(
+	repo_id="bartowski/Phi-3.5-mini-instruct-GGUF",
+	filename="Phi-3.5-mini-instruct-IQ3_XS.gguf",
+    n_gpu_layers=-1,
+)
+
+def chatmsg(message, role):
+    return {"role": role, "content": message}
+
+
+def conv(msgs:List[Dict[str, str]]):
+    return "\n".join([f"<|{msg['role']}|>\n{msg['content']}<|end|>" for msg in msgs])
+
+with gr.Blocks() as demo:
+    chatbot = gr.Chatbot([chatmsg("What is the capital?","user")],type="messages")
+    msg = gr.Textbox()
+    submit = gr.Button("Submit")
+    clear = gr.ClearButton([msg, chatbot])
+
+    def respond(message, chat_history:List[Dict[str, str]]):
+        if "End of conversation." in [i["content"] for i in chat_history]:
+            return "", chat_history
+        chat_history.append(chatmsg(message,"assistant"))
+        ret=llm(syspropmt+conv(chat_history)+"<|user|>\n", stop=["\n \n","?\n",".\n","tile|>"],max_tokens=100)#Only stop at tile|> to see if the conv got terminated
+        comp=ret["choices"][0]["text"]
+        print(repr(comp))
+        if("<|end" in comp):
+            chat_history.append(chatmsg("End of conversation.","user"))
+        else:
+            chat_history.append(chatmsg(comp,"user"))
+        return "", chat_history
+
+    submit.click(respond, [msg, chatbot], [msg, chatbot])
+    msg.submit(respond, [msg, chatbot], [msg, chatbot])
+
+demo.launch()

requirements.txt

@@ -0,0 +1 @@
+llama-cpp-python