Add get human input

.gitignore

@@ -0,0 +1,174 @@
+.docusaurus/
+node_modules/
+# Project
+/.vs
+.vscode
+
+# Log files
+*.log
+
+# Python virtualenv
+.venv
+
+# Byte-compiled / optimized / DLL files
+__pycache__/
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build/
+develop-eggs/
+dist/
+downloads/
+eggs/
+.eggs/
+lib/
+lib64/
+parts/
+sdist/
+var/
+wheels/
+share/python-wheels/
+*.egg-info/
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Unit test / coverage reports
+htmlcov/
+.tox/
+.nox/
+.coverage
+.coverage.*
+.cache
+nosetests.xml
+coverage.xml
+*.cover
+*.py,cover
+.hypothesis/
+.pytest_cache/
+cover/
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+db.sqlite3-journal
+
+# Flask stuff:
+instance/
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build/
+
+# PyBuilder
+.pybuilder/
+target/
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default/
+ipython_config.py
+
+# pyenv
+#   For a library or package, you might want to ignore these files since the code is
+#   intended to run in multiple environments; otherwise, check them in:
+# .python-version
+
+# pipenv
+#   According to pypa/pipenv#598, it is recommended to include Pipfile.lock in version control.
+#   However, in case of collaboration, if having platform-specific dependencies or dependencies
+#   having no cross-platform support, pipenv may install dependencies that don't work, or not
+#   install all needed dependencies.
+#Pipfile.lock
+
+# PEP 582; used by e.g. github.com/David-OConnor/pyflow
+__pypackages__/
+
+# Celery stuff
+celerybeat-schedule
+celerybeat.pid
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mkdocs documentation
+/site
+
+# mypy
+.mypy_cache/
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre/
+
+# pytype static type analyzer
+.pytype/
+
+# Cython debug symbols
+cython_debug/
+
+logs
+
+.idea/*
+.DS_Store
+
+output/
+*.pkl
+
+# local config files
+*.config.local
+OAI_CONFIG_LIST
+key_openai.txt
+key_aoai.txt
+base_aoai.txt
+wolfram.txt
+
+# DB on disk for TeachableAgent
+tmp/
+
+.cache
+test*
+coding
+.chromadb

app.py

@@ -1,24 +1,23 @@
+import asyncio
 import os
 import random
 import time
+from functools import partial
 
 import autogen
 import panel as pn
 from autogen_utils import (
     MathUserProxyAgent,
     RetrieveUserProxyAgent,
+    check_termination_and_human_reply,
     get_retrieve_config,
     initialize_agents,
 )
+from configs import Q1, Q2, Q3, TIMEOUT, TITLE
 from custom_widgets import RowAgentWidget
 from panel.chat import ChatInterface
-from panel.widgets import Button, PasswordInput, Switch, TextInput
+from panel.widgets import Button, PasswordInput, Switch, TextAreaInput, TextInput
 
-TIMEOUT = 60
-TITLE = "Microsoft AutoGen Playground"
-Q1 = "What's AutoGen?"
-Q2 = "Write a python function to compute the sum of numbers."
-Q3 = "find papers on LLM applications from arxiv in the last week, create a markdown table of different domains."
 pn.extension(design="material")
 
 template = pn.template.BootstrapTemplate(title=TITLE)
@@ -199,7 +198,6 @@ async def agents_chat(init_sender, manager, contents, agents):
 
 
 async def reply_chat(contents, user, instance):
-    # print([message for message in instance.objects])
     if hasattr(instance, "collection_name"):
         collection_name = instance.collection_name
     else:
@@ -224,6 +222,15 @@ async def reply_chat(contents, user, instance):
         if "UserProxy" in str(type(agent)):
             init_sender = agent
             break
+    for agent in agents:
+        # Hack for get human input
+        agent._reply_func_list.pop(1)
+        agent.register_reply(
+            [autogen.Agent, None],
+            partial(check_termination_and_human_reply, instance=instance),
+            1,
+        )
+
     if not init_sender:
         init_sender = agents[0]
     await agents_chat(init_sender, manager, contents, agents)

autogen_utils.py

@@ -1,10 +1,13 @@
+import asyncio
 import sys
 import threading
+import time
 from ast import literal_eval
 
 import autogen
 import chromadb
-from autogen import AssistantAgent, UserProxyAgent
+import panel as pn
+from autogen import Agent, AssistantAgent, UserProxyAgent
 from autogen.agentchat.contrib.compressible_agent import CompressibleAgent
 from autogen.agentchat.contrib.gpt_assistant_agent import GPTAssistantAgent
 from autogen.agentchat.contrib.llava_agent import LLaVAAgent
@@ -13,6 +16,15 @@ from autogen.agentchat.contrib.retrieve_assistant_agent import RetrieveAssistant
 from autogen.agentchat.contrib.retrieve_user_proxy_agent import RetrieveUserProxyAgent
 from autogen.agentchat.contrib.teachable_agent import TeachableAgent
 from autogen.code_utils import extract_code
+from configs import Q1, Q2, Q3, TIMEOUT, TITLE
+from panel.widgets import TextAreaInput
+
+try:
+    from termcolor import colored
+except ImportError:
+
+    def colored(x, *args, **kwargs):
+        return x
 
 
 def get_retrieve_config(docs_path, model_name, collection_name):
@@ -29,7 +41,9 @@ def get_retrieve_config(docs_path, model_name, collection_name):
 
 # autogen.ChatCompletion.start_logging()
 def termination_msg(x):
-    return isinstance(x, dict) and "TERMINATE" == str(x.get("content", ""))[-9:].upper()
+    """Check if a message is a termination message."""
+    _msg = str(x.get("content", "")).upper().strip().strip("\n").strip(".")
+    return isinstance(x, dict) and (_msg.endswith("TERMINATE") or _msg.startswith("TERMINATE"))
 
 
 def _is_termination_msg(message):
@@ -50,6 +64,37 @@ def _is_termination_msg(message):
     return not contain_code
 
 
+def new_generate_oai_reply(
+    self,
+    messages=None,
+    sender=None,
+    config=None,
+):
+    """Generate a reply using autogen.oai."""
+    client = self.client if config is None else config
+    if client is None:
+        return False, None
+    if messages is None:
+        messages = self._oai_messages[sender]
+
+    # handle 336006 https://cloud.baidu.com/doc/WENXINWORKSHOP/s/tlmyncueh
+    _context = messages[-1].pop("context", None)
+    _messages = self._oai_system_message + messages
+    for idx, msg in enumerate(_messages):
+        if idx == 0:
+            continue
+        if idx % 2 == 1:
+            msg["role"] = "user" if msg.get("role") != "function" else "function"
+        else:
+            msg["role"] = "assistant"
+    if len(_messages) % 2 == 1:
+        _messages.append({"content": "Please reply exactly `TERMINATE` to me if the task is done.", "role": "user"})
+    # print(f"messages: {_messages}")
+    response = client.create(context=_context, messages=_messages)
+    # print(f"{response=}")
+    return True, client.extract_text_or_function_call(response)[0]
+
+
 def initialize_agents(
     llm_config, agent_name, system_msg, agent_type, retrieve_config=None, code_execution_config=False
 ):
@@ -57,11 +102,11 @@ def initialize_agents(
         agent = RetrieveUserProxyAgent(
             name=agent_name,
             is_termination_msg=termination_msg,
-            human_input_mode="NEVER",
+            human_input_mode="TERMINATE",
             max_consecutive_auto_reply=5,
             retrieve_config=retrieve_config,
             code_execution_config=code_execution_config,  # set to False if you don't want to execute the code
-            default_auto_reply="Reply `TERMINATE` if the task is done.",
+            default_auto_reply="Please reply exactly `TERMINATE` to me if the task is done.",
         )
     elif "GPTAssistantAgent" == agent_type:
         agent = GPTAssistantAgent(
@@ -88,9 +133,9 @@ def initialize_agents(
         agent = globals()[agent_type](
             name=agent_name,
             is_termination_msg=termination_msg,
-            human_input_mode="NEVER",
+            human_input_mode="TERMINATE",
             system_message=system_msg,
-            default_auto_reply="Reply `TERMINATE` if the task is done.",
+            default_auto_reply="Please reply exactly `TERMINATE` to me if the task is done.",
             max_consecutive_auto_reply=5,
             code_execution_config=code_execution_config,
         )
@@ -102,6 +147,114 @@ def initialize_agents(
             system_message=system_msg,
             llm_config=llm_config,
         )
-
+    # if any(["ernie" in cfg["model"].lower() for cfg in llm_config["config_list"]]):
+    if "ernie" in llm_config["config_list"][0]["model"].lower():
+        # Hack for ERNIE Bot models
+        # print("Hack for ERNIE Bot models.")
+        agent._reply_func_list.pop(-1)
+        agent.register_reply([Agent, None], new_generate_oai_reply, -1)
     return agent
 
+
+async def get_human_input(name, prompt: str, instance=None) -> str:
+    """Get human input."""
+    if instance is None:
+        return input(prompt)
+    get_input_widget = TextAreaInput(placeholder=prompt, name="", sizing_mode="stretch_width")
+    get_input_checkbox = pn.widgets.Checkbox(name="Check to Submit Feedback")
+    instance.send(pn.Row(get_input_widget, get_input_checkbox), user=name, respond=False)
+    ts = time.time()
+    while True:
+        if time.time() - ts > TIMEOUT:
+            instance.send(
+                f"You didn't provide your feedback in {TIMEOUT} seconds, skip and use auto-reply.",
+                user=name,
+                respond=False,
+            )
+            reply = ""
+            break
+        if get_input_widget.value != "" and get_input_checkbox.value is True:
+            get_input_widget.disabled = True
+            reply = get_input_widget.value
+            break
+        await asyncio.sleep(0.1)
+    return reply
+
+
+async def check_termination_and_human_reply(
+    self,
+    messages=None,
+    sender=None,
+    config=None,
+    instance=None,
+):
+    """Check if the conversation should be terminated, and if human reply is provided."""
+    if config is None:
+        config = self
+    if messages is None:
+        messages = self._oai_messages[sender]
+    message = messages[-1]
+    reply = ""
+    no_human_input_msg = ""
+    if self.human_input_mode == "ALWAYS":
+        reply = await get_human_input(
+            self.name,
+            f"Provide feedback to {sender.name}. Press enter to skip and use auto-reply, or type 'exit' to end the conversation: ",
+            instance,
+        )
+        no_human_input_msg = "NO HUMAN INPUT RECEIVED." if not reply else ""
+        # if the human input is empty, and the message is a termination message, then we will terminate the conversation
+        reply = reply if reply or not self._is_termination_msg(message) else "exit"
+    else:
+        if self._consecutive_auto_reply_counter[sender] >= self._max_consecutive_auto_reply_dict[sender]:
+            if self.human_input_mode == "NEVER":
+                reply = "exit"
+            else:
+                # self.human_input_mode == "TERMINATE":
+                terminate = self._is_termination_msg(message)
+                reply = await get_human_input(
+                    self.name,
+                    f"Please give feedback to {sender.name}. Press enter or type 'exit' to stop the conversation: "
+                    if terminate
+                    else f"Please give feedback to {sender.name}. Press enter to skip and use auto-reply, or type 'exit' to stop the conversation: ",
+                    instance,
+                )
+                no_human_input_msg = "NO HUMAN INPUT RECEIVED." if not reply else ""
+                # if the human input is empty, and the message is a termination message, then we will terminate the conversation
+                reply = reply if reply or not terminate else "exit"
+        elif self._is_termination_msg(message):
+            if self.human_input_mode == "NEVER":
+                reply = "exit"
+            else:
+                # self.human_input_mode == "TERMINATE":
+                reply = await get_human_input(
+                    self.name,
+                    f"Please give feedback to {sender.name}. Press enter or type 'exit' to stop the conversation: ",
+                    instance,
+                )
+                no_human_input_msg = "NO HUMAN INPUT RECEIVED." if not reply else ""
+                # if the human input is empty, and the message is a termination message, then we will terminate the conversation
+                reply = reply or "exit"
+
+    # print the no_human_input_msg
+    if no_human_input_msg:
+        print(colored(f"\n>>>>>>>> {no_human_input_msg}", "red"), flush=True)
+
+    # stop the conversation
+    if reply == "exit":
+        # reset the consecutive_auto_reply_counter
+        self._consecutive_auto_reply_counter[sender] = 0
+        return True, None
+
+    # send the human reply
+    if reply or self._max_consecutive_auto_reply_dict[sender] == 0:
+        # reset the consecutive_auto_reply_counter
+        self._consecutive_auto_reply_counter[sender] = 0
+        return True, reply
+
+    # increment the consecutive_auto_reply_counter
+    self._consecutive_auto_reply_counter[sender] += 1
+    if self.human_input_mode != "NEVER":
+        print(colored("\n>>>>>>>> USING AUTO REPLY...", "red"), flush=True)
+
+    return False, None

configs.py

@@ -0,0 +1,5 @@
+TIMEOUT = 60
+TITLE = "Microsoft AutoGen Playground"
+Q1 = "What's AutoGen?"
+Q2 = "Write a python function to compute the sum of numbers."
+Q3 = "find papers on LLM applications from arxiv in the last week, create a markdown table of different domains."