open-strawberry

app.py

@@ -0,0 +1,350 @@
+import os
+
+import openai
+
+import streamlit as st
+import time
+try:
+    from src.models import get_all_model_names
+    from src.open_strawberry import get_defaults, manage_conversation
+except (ModuleNotFoundError, ImportError):
+    from models import get_all_model_names
+    from open_strawberry import get_defaults, manage_conversation
+
+(model, system_prompt, initial_prompt, expected_answer,
+ next_prompts, num_turns, show_next, final_prompt,
+ temperature, max_tokens,
+ num_turns_final_mod,
+ show_cot,
+ verbose) = get_defaults()
+
+st.title("Open Strawberry Conversation")
+st.markdown("[Open Strawberry GitHub Repo](https://github.com/pseudotensor/open-strawberry)")
+
+# Initialize session state
+if "messages" not in st.session_state:
+    st.session_state.messages = []
+if "turn_count" not in st.session_state:
+    st.session_state.turn_count = 0
+if "input_key" not in st.session_state:
+    st.session_state.input_key = 0
+if "conversation_started" not in st.session_state:
+    st.session_state.conversation_started = False
+if "waiting_for_continue" not in st.session_state:
+    st.session_state.waiting_for_continue = False
+if "generator" not in st.session_state:
+    st.session_state.generator = None  # Store the generator in session state
+if "prompt" not in st.session_state:
+    st.session_state.prompt = None  # Store the prompt in session state
+if "answer" not in st.session_state:
+    st.session_state.answer = None
+if "system_prompt" not in st.session_state:
+    st.session_state.system_prompt = None
+if "output_tokens" not in st.session_state:
+    st.session_state.output_tokens = 0
+if "input_tokens" not in st.session_state:
+    st.session_state.input_tokens = 0
+if "cache_creation_input_tokens" not in st.session_state:
+    st.session_state.cache_creation_input_tokens = 0
+if "cache_read_input_tokens" not in st.session_state:
+    st.session_state.cache_read_input_tokens = 0
+if "verbose" not in st.session_state:
+    st.session_state.verbose = verbose
+if "max_tokens" not in st.session_state:
+    st.session_state.max_tokens = max_tokens
+if "temperature" not in st.session_state:
+    st.session_state.temperature = temperature
+if "next_prompts" not in st.session_state:
+    st.session_state.next_prompts = next_prompts
+if "final_prompt" not in st.session_state:
+    st.session_state.final_prompt = final_prompt
+
+
+# Function to display chat messages
+def display_chat():
+    display_step = 1
+    for message in st.session_state.messages:
+        if message["role"] == "assistant":
+            if 'final' in message and message['final']:
+                display_final(message)
+            elif 'turn_title' in message and message['turn_title']:
+                display_turn_title(message, display_step=display_step)
+                display_step += 1
+            else:
+                with st.expander("Chain of Thoughts", expanded=st.session_state["show_cot"]):
+                    assistant_container1 = st.chat_message("assistant")
+                    with assistant_container1.container():
+                        st.markdown(message["content"].replace('\n', '  \n'), unsafe_allow_html=True)
+        elif message["role"] == "user":
+            if not message["initial"] and not st.session_state.show_next:
+                continue
+            user_container1 = st.chat_message("user")
+            with user_container1:
+                st.markdown(message["content"].replace('\n', '  \n'), unsafe_allow_html=True)
+
+
+def display_final(chunk1, can_rerun=False):
+    if 'final' in chunk1 and chunk1['final']:
+        if st.session_state.answer:
+            if st.session_state.answer.strip() in chunk1["content"]:
+                st.markdown(f'<h3 class="expander-title">🏆 Final Answer</h3>', unsafe_allow_html=True)
+            else:
+                st.markdown(f'Expected: **{st.session_state.answer.strip()}**', unsafe_allow_html=True)
+                st.markdown(f'<h3 class="expander-title">👎 Final Answer</h3>', unsafe_allow_html=True)
+        else:
+            st.markdown(f'<h3 class="expander-title">👌 Final Answer</h3>', unsafe_allow_html=True)
+        final = chunk1["content"].strip().replace('\n', '  \n')
+        if '\n' in final or '<br>' in final:
+            st.markdown(f'{final}', unsafe_allow_html=True)
+        else:
+            st.markdown(f'**{final}**', unsafe_allow_html=True)
+        if can_rerun:
+            # rerun to get token stats
+            st.rerun()
+
+
+def display_turn_title(chunk1, display_step=None):
+    if display_step is None:
+        display_step = st.session_state.turn_count
+        name = "Completed Step"
+    else:
+        name = "Step"
+    if 'turn_title' in chunk1 and chunk1['turn_title']:
+        turn_title = chunk1["content"].strip().replace('\n', '  \n')
+        step_time = f' in time {str(int(chunk1["thinking_time"]))}s'
+        acum_time = f' in total {str(int(chunk1["total_thinking_time"]))}s'
+        st.markdown(f'**{name} {display_step}: {turn_title}{step_time}{acum_time}**', unsafe_allow_html=True)
+
+
+if st.button("Start Conversation", disabled=st.session_state.conversation_started):
+    st.session_state.conversation_started = True
+
+# Sidebar
+st.sidebar.title("Controls")
+
+on_hf_spaces = os.getenv("HF_SPACES", '0') == '1'
+
+
+def save_env_vars(env_vars):
+    assert not on_hf_spaces, "Cannot save env vars in HF Spaces"
+    env_path = os.path.join(os.path.dirname(__file__), "..", ".env")
+    from dotenv import set_key
+    for key, value in env_vars.items():
+        set_key(env_path, key, value)
+
+
+def get_dotenv_values():
+    if on_hf_spaces:
+        return st.session_state.secrets
+    else:
+        from dotenv import dotenv_values
+        return dotenv_values(os.path.join(os.path.dirname(__file__), "..", ".env"))
+
+
+if 'secrets' not in st.session_state:
+    if on_hf_spaces:
+        # allow user to enter
+        st.session_state.secrets = dict(OPENAI_API_KEY='',
+                                        OPENAI_BASE_URL='https://api.openai.com/v1',
+                                        OPENAI_MODEL_NAME='',
+                                        # OLLAMA_OPENAI_API_KEY='',
+                                        # OLLAMA_OPENAI_BASE_URL='http://localhost:11434/v1/',
+                                        # OLLAMA_OPENAI_MODEL_NAME='',
+                                        # AZURE_OPENAI_API_KEY='',
+                                        # AZURE_OPENAI_API_VERSION='',
+                                        # AZURE_OPENAI_ENDPOINT='',
+                                        # AZURE_OPENAI_DEPLOYMENT='',
+                                        # AZURE_OPENAI_MODEL_NAME='',
+                                        GEMINI_API_KEY='',
+                                        # MISTRAL_API_KEY='',
+                                        GROQ_API_KEY='',
+                                        ANTHROPIC_API_KEY='',
+                                        )
+
+    else:
+        st.session_state.secrets = {}
+
+
+def update_model_selection():
+    visible_models1 = get_all_model_names(st.session_state.secrets, on_hf_spaces)
+    if visible_models1 and "model_name" in st.session_state:
+        if st.session_state.model_name not in visible_models1:
+            st.session_state.model_name = visible_models1[0]
+
+
+# Replace the existing model selection code with this
+if 'model_name' not in st.session_state or not st.session_state.model_name:
+    update_model_selection()
+
+# Model selection
+visible_models = get_all_model_names(st.session_state.secrets, on_hf_spaces)
+st.sidebar.selectbox("Select Model", visible_models, key="model_name",
+                     disabled=st.session_state.conversation_started)
+st.sidebar.checkbox("Show Next", value=show_next, key="show_next", disabled=st.session_state.conversation_started)
+st.sidebar.number_input("Num Turns to Check if Final Answer", value=num_turns_final_mod, key="num_turns_final_mod",
+                        disabled=st.session_state.conversation_started)
+st.sidebar.number_input("Num Turns per User Click of Continue", value=num_turns, key="num_turns",
+                        disabled=st.session_state.conversation_started)
+st.sidebar.checkbox("Show Chain of Thoughts Details", value=show_cot, key="show_cot",
+                    disabled=st.session_state.conversation_started)
+
+# Reset conversation button
+reset_clicked = st.sidebar.button("Reset Conversation")
+with st.sidebar.expander("Edit in-memory session secrets" if on_hf_spaces else "Edit .env", expanded=on_hf_spaces):
+    dotenv_dict = get_dotenv_values()
+    new_env = {}
+    for k, v in dotenv_dict.items():
+        new_env[k] = st.text_input(k, value=v, key=k, disabled=st.session_state.conversation_started, type="password")
+        st.session_state.secrets[k] = new_env[k]
+    save_secrets_clicked = st.button("Save dotenv" if not on_hf_spaces else "Save secrets to memory")
+
+    if save_secrets_clicked:
+        if on_hf_spaces:
+            st.success("secrets temporarily stored to your session memory only")
+        else:
+            save_env_vars(st.session_state.user_secrets)
+            st.success("dotenv saved to .env file")
+
+if reset_clicked:
+    st.session_state.messages = []
+    st.session_state.turn_count = 0
+    st.sidebar.write(f"Turn count: {st.session_state.turn_count}")
+    st.session_state.input_key += 1
+    st.session_state.conversation_started = False
+    st.session_state.generator = None  # Reset the generator
+    reset_clicked = False
+    st.session_state.output_tokens = 0
+    st.session_state.input_tokens = 0
+    st.session_state.cache_creation_input_tokens = 0
+    st.session_state.cache_read_input_tokens = 0
+    st.rerun()
+
+st.session_state.waiting_for_continue = False
+
+# Display debug information
+st.sidebar.write(f"Turn count: {st.session_state.turn_count}")
+num_messages = len([x for x in st.session_state.messages if x.get('role', '') == 'assistant'])
+st.sidebar.write(f"Number of AI messages: {num_messages}")
+st.sidebar.write(f"Conversation started: {st.session_state.conversation_started}")
+st.sidebar.write(f"Output tokens: {st.session_state.output_tokens}")
+st.sidebar.write(f"Input tokens: {st.session_state.input_tokens}")
+st.sidebar.write(f"Cache creation input tokens: {st.session_state.cache_creation_input_tokens}")
+st.sidebar.write(f"Cache read input tokens: {st.session_state.cache_read_input_tokens}")
+
+# Handle user input
+if not st.session_state.conversation_started:
+    prompt = st.text_area("What would you like to ask?", value=initial_prompt,
+                          key=f"input_{st.session_state.input_key}", height=500)
+    st.session_state.prompt = prompt
+    answer = st.text_area("Expected answer (Empty if do not know)", value=expected_answer,
+                          key=f"answer_{st.session_state.input_key}", height=100)
+    st.session_state.answer = answer
+    system_prompt = st.text_area("System Prompt", value=system_prompt,
+                                 key=f"system_prompt_{st.session_state.input_key}", height=200)
+    st.session_state.system_prompt = system_prompt
+else:
+    st.session_state.conversation_started = True
+    st.session_state.input_key += 1
+
+# Display chat history
+chat_container = st.container()
+with chat_container:
+    display_chat()
+
+# Process conversation
+current_assistant_message = ''
+assistant_placeholder = None
+
+try:
+    while True:
+        if st.session_state.waiting_for_continue:
+            time.sleep(0.1)  # Short sleep to prevent excessive CPU usage
+            continue
+        if not st.session_state.conversation_started:
+            time.sleep(0.1)
+            continue
+        elif st.session_state.generator is None:
+            st.session_state.generator = manage_conversation(
+                model=st.session_state["model_name"],
+                system=st.session_state.system_prompt,
+                initial_prompt=st.session_state.prompt,
+                next_prompts=st.session_state.next_prompts,
+                final_prompt=st.session_state.final_prompt,
+                num_turns_final_mod=st.session_state.num_turns_final_mod,
+                num_turns=st.session_state.num_turns,
+                temperature=st.session_state.temperature,
+                max_tokens=st.session_state.max_tokens,
+                verbose=st.session_state.verbose,
+            )
+        chunk = next(st.session_state.generator)
+        if chunk["role"] == "assistant":
+            if not chunk.get('final', False) and not chunk.get('turn_title', False):
+                current_assistant_message += chunk["content"]
+            if assistant_placeholder is None:
+                assistant_placeholder = st.empty()  # Placeholder for assistant's message
+
+            # Update the assistant container with the progressively streaming message
+            with assistant_placeholder.container():
+                # Update in the same chat message
+                with st.expander("Chain of Thoughts", expanded=st.session_state["show_cot"]):
+                    st.chat_message("assistant").markdown(current_assistant_message, unsafe_allow_html=True)
+                if 'turn_title' in chunk and chunk['turn_title']:
+                    st.session_state.messages.append(
+                        {"role": "assistant", "content": chunk['content'], 'turn_title': True,
+                         'thinking_time': chunk['thinking_time'],
+                         'total_thinking_time': chunk['total_thinking_time']})
+                    display_turn_title(chunk)
+                if 'final' in chunk and chunk['final']:
+                    # user role would normally do this, but on final step needs to be here
+                    st.session_state.messages.append(
+                        {"role": "assistant", "content": current_assistant_message, 'final': False})
+                    # last message, so won't reach user turn, so need to store final assistant message from parsing
+                    st.session_state.messages.append(
+                        {"role": "assistant", "content": chunk['content'], 'final': True})
+                    display_final(chunk, can_rerun=True)
+
+        elif chunk["role"] == "user":
+            if current_assistant_message:
+                st.session_state.messages.append(
+                    {"role": "assistant", "content": current_assistant_message, 'final': chunk.get('final', False)})
+            # Reset assistant message when user provides input
+            # Display user message
+            if not chunk["initial"] and not st.session_state.show_next:
+                pass
+            else:
+                user_container = st.chat_message("user")
+                with user_container:
+                    st.markdown(chunk["content"].replace('\n', '  \n'), unsafe_allow_html=True)
+            st.session_state.messages.append({"role": "user", "content": chunk["content"], 'initial': chunk["initial"]})
+
+            st.session_state.turn_count += 1
+            if current_assistant_message:
+                assistant_placeholder = st.empty()  # Reset placeholder
+                current_assistant_message = ""
+
+        elif chunk["role"] == "action":
+            if chunk["content"] in ["continue?"]:
+                # Continue conversation button
+                continue_clicked = st.button("Continue Conversation")
+                st.session_state.waiting_for_continue = True
+            st.session_state.turn_count += 1
+            if current_assistant_message:
+                st.session_state.messages.append({"role": "assistant", "content": current_assistant_message})
+                assistant_placeholder = st.empty()  # Reset placeholder
+                current_assistant_message = ""
+            elif chunk["content"] == "end":
+                break
+        elif chunk["role"] == "usage":
+            st.session_state.output_tokens += chunk["content"]["output_tokens"] if "output_tokens" in chunk[
+                "content"] else 0
+            st.session_state.input_tokens += chunk["content"]["input_tokens"] if "input_tokens" in chunk[
+                "content"] else 0
+            st.session_state.cache_creation_input_tokens += chunk["content"][
+                "cache_creation_input_tokens"] if "cache_creation_input_tokens" in chunk["content"] else 0
+            st.session_state.cache_read_input_tokens += chunk["content"][
+                "cache_read_input_tokens"] if "cache_read_input_tokens" in chunk["content"] else 0
+
+        time.sleep(0.001)  # Small delay to prevent excessive updates
+
+except StopIteration:
+    pass

cli.py

@@ -0,0 +1,107 @@
+import argparse
+import time
+
+from src.open_strawberry import get_defaults, manage_conversation
+
+
+def parse_arguments(model, system_prompt, next_prompts, num_turns, show_next, final_prompt,
+                    num_turns_final_mod, show_cot, verbose):
+    parser = argparse.ArgumentParser(description="Open Strawberry Conversation Manager")
+    parser.add_argument("--show_next", action="store_true", default=show_next, help="Show all messages")
+    parser.add_argument("--verbose", action="store_true", default=verbose, help="Show usage information")
+    parser.add_argument("--system_prompt", type=str, default=system_prompt, help="Custom system prompt")
+    parser.add_argument("--num_turns_final_mod", type=int, default=num_turns_final_mod,
+                        help="Number of turns before final prompt")
+    parser.add_argument("--num_turns", type=int, default=num_turns,
+                        help="Number of turns before pausing for continuation")
+    parser.add_argument("--model", type=str, default=model, help="Model to use for conversation")
+    parser.add_argument("--initial_prompt", type=str, default='', help="Initial prompt.  If empty, then ask user.")
+    parser.add_argument("--expected_answer", type=str, default='', help="Expected answer.  If empty, then ignore.")
+    parser.add_argument("--next_prompts", type=str, nargs="+", default=next_prompts, help="Next prompts")
+    parser.add_argument("--final_prompt", type=str, default=final_prompt, help="Final prompt")
+    parser.add_argument("--temperature", type=float, default=0.3, help="Temperature for the model")
+    parser.add_argument("--max_tokens", type=int, default=1024, help="Maximum tokens for the model")
+    parser.add_argument("--seed", type=int, default=0, help="Random seed, 0 means random seed")
+    parser.add_argument("--show_cot", type=bool, default=show_cot, help="Whether to show detailed Chain of Thoughts")
+
+    return parser.parse_args()
+
+
+def go_cli():
+    (model, system_prompt, initial_prompt, expected_answer,
+     next_prompts, num_turns, show_next, final_prompt,
+     temperature, max_tokens, num_turns_final_mod,
+     show_cot, verbose) = get_defaults()
+    args = parse_arguments(model, system_prompt, next_prompts, num_turns, show_next, final_prompt,
+                           num_turns_final_mod, show_cot, verbose)
+
+    if args.initial_prompt == '':
+        initial_prompt_query = input("Enter the initial prompt (hitting enter will use default initial_prompt)\n\n")
+        if initial_prompt_query not in ['', '\n', '\r\n']:
+            initial_prompt_chosen = initial_prompt_query
+        else:
+            initial_prompt_chosen = initial_prompt
+    else:
+        initial_prompt_chosen = args.initial_prompt
+
+    generator = manage_conversation(model=args.model,
+                                    system=args.system_prompt,
+                                    initial_prompt=initial_prompt_chosen,
+                                    next_prompts=args.next_prompts,
+                                    final_prompt=args.final_prompt,
+                                    num_turns_final_mod=args.num_turns_final_mod,
+                                    num_turns=args.num_turns,
+                                    temperature=args.temperature,
+                                    max_tokens=args.max_tokens,
+                                    seed=args.seed,
+                                    cli_mode=True)
+    response = ''
+    conversation_history = []
+
+    try:
+        step = 1
+        while True:
+            chunk = next(generator)
+            if 'role' in chunk and chunk['role'] == 'assistant':
+                response += chunk['content']
+
+                if 'turn_title' in chunk and chunk['turn_title']:
+                    step_time = f' in time {str(int(chunk["thinking_time"]))}s'
+                    acum_time = f' in total {str(int(chunk["total_thinking_time"]))}s'
+                    extra = '\n\n' if show_cot else ''
+                    extra2 = '**' if show_cot else ''
+                    extra3 = ' ' if show_cot else ''
+                    print(
+                        f'{extra}{extra2}{extra3}Completed Step {step}: {chunk["content"]}{step_time}{acum_time}{extra3}{extra2}{extra}')
+                    step += 1
+                elif 'final' in chunk and chunk['final']:
+                    if '\n' in chunk['content'] or '\r' in chunk['content']:
+                        print(f'\n\nFinal Answer:\n\n {chunk["content"]}')
+                    else:
+                        print('\n\nFinal Answer:\n\n**', chunk['content'], '**\n\n')
+                elif show_cot:
+                    print(chunk['content'], end='')
+                if 'chat_history' in chunk:
+                    conversation_history = chunk['chat_history']
+            elif 'role' in chunk and chunk['role'] == 'user':
+                if not chunk['initial'] and not show_next:
+                    if show_cot:
+                        print('\n\n')
+                    continue
+                print('\n', end='')  # finish assistant
+                print('\nUser: ', chunk['content'], end='\n\n')
+                print('\nAssistant:\n\n ')
+            time.sleep(0.001)
+    except StopIteration as e:
+        pass
+
+    if verbose:
+        print("Conversation history:", conversation_history)
+
+    if expected_answer and expected_answer in conversation_history[-1]['content']:
+        print("\n\nGot Expected answer!")
+
+    if not show_cot:
+        print("**FULL RESPONSE:**\n\n")
+        print(response)
+    return response

models.py

@@ -0,0 +1,483 @@
+import ast
+import datetime
+import os
+from typing import List, Dict, Generator
+from dotenv import load_dotenv
+
+from tenacity import (
+    retry,
+    stop_after_attempt,
+    wait_random_exponential,
+)  # for exponential backoff
+
+# Load environment variables from .env file
+load_dotenv()
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(3))
+def anthropic_completion_with_backoff(client, *args, **kwargs):
+    return client.beta.prompt_caching.messages.create(*args, **kwargs)
+
+
+def get_anthropic(model: str,
+                  prompt: str,
+                  temperature: float = 0,
+                  max_tokens: int = 4096,
+                  system: str = '',
+                  chat_history: List[Dict] = None,
+                  verbose=False) -> \
+        Generator[dict, None, None]:
+    model = model.replace('anthropic:', '')
+
+    # https://docs.anthropic.com/en/docs/build-with-claude/prompt-caching
+    import anthropic
+
+    clawd_key = os.getenv('ANTHROPIC_API_KEY')
+    clawd_client = anthropic.Anthropic(api_key=clawd_key) if clawd_key else None
+
+    if chat_history is None:
+        chat_history = []
+
+    messages = []
+
+    # Add conversation history, removing cache_control from all but the last two user messages
+    for i, message in enumerate(chat_history):
+        if message["role"] == "user":
+            if i >= len(chat_history) - 3:  # Last two user messages
+                messages.append(message)
+            else:
+                messages.append({
+                    "role": "user",
+                    "content": [{"type": "text", "text": message["content"][0]["text"]}]
+                })
+        else:
+            messages.append(message)
+
+    # Add the new user message
+    messages.append({
+        "role": "user",
+        "content": [
+            {
+                "type": "text",
+                "text": prompt,
+                "cache_control": {"type": "ephemeral"}
+            }
+        ]
+    })
+
+    response = anthropic_completion_with_backoff(clawd_client,
+                                                 model=model,
+                                                 max_tokens=max_tokens,
+                                                 temperature=temperature,
+                                                 system=system,
+                                                 messages=messages,
+                                                 stream=True
+                                                 )
+
+    output_tokens = 0
+    input_tokens = 0
+    cache_creation_input_tokens = 0
+    cache_read_input_tokens = 0
+    for chunk in response:
+        if chunk.type == "content_block_start":
+            # This is where we might find usage info in the future
+            pass
+        elif chunk.type == "content_block_delta":
+            yield dict(text=chunk.delta.text)
+        elif chunk.type == "message_delta":
+            output_tokens = dict(chunk.usage).get('output_tokens', 0)
+        elif chunk.type == "message_start":
+            usage = chunk.message.usage
+            input_tokens = dict(usage).get('input_tokens', 0)
+            cache_creation_input_tokens = dict(usage).get('cache_creation_input_tokens', 0)
+            cache_read_input_tokens = dict(usage).get('cache_read_input_tokens', 0)
+        else:
+            if verbose:
+                print("Unknown chunk type:", chunk.type)
+                print("Chunk:", chunk)
+
+    if verbose:
+        # After streaming is complete, print the usage information
+        print(f"Output tokens: {output_tokens}")
+        print(f"Input tokens: {input_tokens}")
+        print(f"Cache creation input tokens: {cache_creation_input_tokens}")
+        print(f"Cache read input tokens: {cache_read_input_tokens}")
+    yield dict(output_tokens=output_tokens, input_tokens=input_tokens,
+               cache_creation_input_tokens=cache_creation_input_tokens,
+               cache_read_input_tokens=cache_read_input_tokens)
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(3))
+def openai_completion_with_backoff(client, *args, **kwargs):
+    return client.chat.completions.create(*args, **kwargs)
+
+
+def get_openai(model: str,
+               prompt: str,
+               temperature: float = 0,
+               max_tokens: int = 4096,
+               system: str = '',
+               chat_history: List[Dict] = None,
+               verbose=False) -> Generator[dict, None, None]:
+    anthropic_models, openai_models, google_models, groq_models, azure_models, ollama = get_model_names()
+    if model in ollama:
+        model = model.replace('ollama:', '')
+        openai_key = os.getenv('OLLAMA_OPENAI_API_KEY')
+        openai_base_url = os.getenv('OLLAMA_OPENAI_BASE_URL', 'http://localhost:11434/v1/')
+    else:
+        model = model.replace('openai:', '')
+        openai_key = os.getenv('OPENAI_API_KEY')
+        openai_base_url = os.getenv('OPENAI_BASE_URL', 'https://api.openai.com/v1')
+
+    from openai import OpenAI
+
+    openai_client = OpenAI(api_key=openai_key, base_url=openai_base_url) if openai_key else None
+
+    if chat_history is None:
+        chat_history = []
+    chat_history_copy = chat_history.copy()
+    for mi, message in enumerate(chat_history_copy):
+        if isinstance(message["content"], list):
+            chat_history_copy[mi]["content"] = message["content"][0]["text"]
+    chat_history = chat_history_copy
+
+    messages = [{"role": "system", "content": system}] + chat_history + [{"role": "user", "content": prompt}]
+
+    response = openai_completion_with_backoff(openai_client,
+                                              model=model,
+                                              messages=messages,
+                                              temperature=temperature,
+                                              max_tokens=max_tokens,
+                                              stream=True,
+                                              )
+
+    output_tokens = 0
+    input_tokens = 0
+    for chunk in response:
+        if chunk.choices[0].delta.content:
+            yield dict(text=chunk.choices[0].delta.content)
+        if chunk.usage:
+            output_tokens = chunk.usage.completion_tokens
+            input_tokens = chunk.usage.prompt_tokens
+
+    if verbose:
+        print(f"Output tokens: {output_tokens}")
+        print(f"Input tokens: {input_tokens}")
+    yield dict(output_tokens=output_tokens, input_tokens=input_tokens)
+
+
+def openai_messages_to_gemini_history(messages):
+    """Converts OpenAI messages to Gemini history format.
+
+    Args:
+        messages: A list of OpenAI messages, each with "role" and "content" keys.
+
+    Returns:
+        A list of dictionaries representing the chat history for Gemini.
+    """
+    history = []
+    for message in messages:
+        if isinstance(message["content"], list):
+            message["content"] = message["content"][0]["text"]
+        if message["role"] == "user":
+            history.append({"role": "user", "parts": [{"text": message["content"]}]})
+        elif message["role"] == "assistant":
+            history.append({"role": "model", "parts": [{"text": message["content"]}]})
+        # Optionally handle system messages if needed
+        # elif message["role"] == "system":
+        #     history.append({"role": "system", "parts": [{"text": message["content"]}]})
+
+    return history
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(3))
+def gemini_send_message_with_backoff(chat, prompt, stream=True):
+    return chat.send_message(prompt, stream=stream)
+
+
+@retry(wait=wait_random_exponential(min=1, max=60), stop=stop_after_attempt(3))
+def gemini_generate_content_with_backoff(model, prompt, stream=True):
+    return model.generate_content(prompt, stream=stream)
+
+
+def get_google(model: str,
+               prompt: str,
+               temperature: float = 0,
+               max_tokens: int = 4096,
+               system: str = '',
+               chat_history: List[Dict] = None,
+               verbose=False) -> Generator[dict, None, None]:
+    model = model.replace('google:', '').replace('gemini:', '')
+
+    import google.generativeai as genai
+
+    gemini_key = os.getenv("GEMINI_API_KEY")
+    genai.configure(api_key=gemini_key)
+    # Create the model
+    generation_config = {
+        "temperature": temperature,
+        "top_p": 0.95,
+        "top_k": 64,
+        "max_output_tokens": max_tokens,
+        "response_mime_type": "text/plain",
+    }
+
+    if chat_history is None:
+        chat_history = []
+
+    chat_history = chat_history.copy()
+    chat_history = openai_messages_to_gemini_history(chat_history)
+
+    # NOTE: assume want own control.  Too many false positives by Google.
+    from google.generativeai.types import HarmCategory
+    from google.generativeai.types import HarmBlockThreshold
+    safety_settings = {
+        HarmCategory.HARM_CATEGORY_DANGEROUS_CONTENT: HarmBlockThreshold.BLOCK_NONE,
+        HarmCategory.HARM_CATEGORY_HATE_SPEECH: HarmBlockThreshold.BLOCK_NONE,
+        HarmCategory.HARM_CATEGORY_HARASSMENT: HarmBlockThreshold.BLOCK_NONE,
+        HarmCategory.HARM_CATEGORY_SEXUALLY_EXPLICIT: HarmBlockThreshold.BLOCK_NONE,
+    }
+
+    cache = None
+    # disable cache for now until work into things well
+    use_cache = False
+    if use_cache and model == 'gemini-1.5-pro':
+        from google.generativeai import caching
+        # Estimate token count (this is a rough estimate, you may need a more accurate method)
+        estimated_tokens = len(prompt.split()) + sum(len(msg['content'].split()) for msg in chat_history)
+
+        if estimated_tokens > 32000:
+            cache = caching.CachedContent.create(
+                model=model,
+                display_name=f'cache_{datetime.datetime.now().isoformat()}',
+                system_instruction=system,
+                contents=[prompt] + [msg['content'] for msg in chat_history],
+                ttl=datetime.timedelta(minutes=5),  # Set an appropriate TTL.  Short for now for cost savings.
+            )
+            gemini_model = genai.GenerativeModel.from_cached_content(cached_content=cache)
+        else:
+            gemini_model = genai.GenerativeModel(model_name=model,
+                                                 generation_config=generation_config,
+                                                 safety_settings=safety_settings)
+    else:
+        gemini_model = genai.GenerativeModel(model_name=model,
+                                             generation_config=generation_config,
+                                             safety_settings=safety_settings)
+
+    if cache:
+        response = gemini_generate_content_with_backoff(gemini_model, prompt, stream=True)
+    else:
+        chat = gemini_model.start_chat(history=chat_history)
+        response = gemini_send_message_with_backoff(chat, prompt, stream=True)
+
+    output_tokens = 0
+    input_tokens = 0
+    cache_read_input_tokens = 0
+    cache_creation_input_tokens = 0
+
+    for chunk in response:
+        if chunk.text:
+            yield dict(text=chunk.text)
+        if chunk.usage_metadata:
+            output_tokens = chunk.usage_metadata.candidates_token_count
+            input_tokens = chunk.usage_metadata.prompt_token_count
+            cache_read_input_tokens = chunk.usage_metadata.cached_content_token_count
+            cache_creation_input_tokens = 0  # This might need to be updated if available in the API
+
+    if verbose:
+        print(f"Output tokens: {output_tokens}")
+        print(f"Input tokens: {input_tokens}")
+        print(f"Cached tokens: {cache_read_input_tokens}")
+
+    yield dict(output_tokens=output_tokens, input_tokens=input_tokens,
+               cache_read_input_tokens=cache_read_input_tokens,
+               cache_creation_input_tokens=cache_creation_input_tokens)
+
+
+def delete_cache(cache):
+    if cache:
+        cache.delete()
+        print(f"Cache {cache.display_name} deleted.")
+    else:
+        print("No cache to delete.")
+
+
+def get_groq(model: str,
+             prompt: str,
+             temperature: float = 0,
+             max_tokens: int = 4096,
+             system: str = '',
+             chat_history: List[Dict] = None,
+             verbose=False) -> Generator[dict, None, None]:
+    model = model.replace('groq:', '')
+
+    from groq import Groq
+
+    groq_key = os.getenv("GROQ_API_KEY")
+    client = Groq(api_key=groq_key)
+
+    if chat_history is None:
+        chat_history = []
+
+    chat_history = chat_history.copy()
+
+    messages = [{"role": "system", "content": system}] + chat_history + [{"role": "user", "content": prompt}]
+
+    stream = openai_completion_with_backoff(client,
+                                            messages=messages,
+                                            model=model,
+                                            temperature=temperature,
+                                            max_tokens=max_tokens,
+                                            stream=True,
+                                            )
+
+    output_tokens = 0
+    input_tokens = 0
+    for chunk in stream:
+        if chunk.choices[0].delta.content:
+            yield dict(text=chunk.choices[0].delta.content)
+        if chunk.usage:
+            output_tokens = chunk.usage.completion_tokens
+            input_tokens = chunk.usage.prompt_tokens
+
+    if verbose:
+        print(f"Output tokens: {output_tokens}")
+        print(f"Input tokens: {input_tokens}")
+    yield dict(output_tokens=output_tokens, input_tokens=input_tokens)
+
+
+def get_openai_azure(model: str,
+                     prompt: str,
+                     temperature: float = 0,
+                     max_tokens: int = 4096,
+                     system: str = '',
+                     chat_history: List[Dict] = None,
+                     verbose=False) -> Generator[dict, None, None]:
+    model = model.replace('azure:', '').replace('openai_azure:', '')
+
+    from openai import AzureOpenAI
+
+    azure_endpoint = os.getenv("AZURE_OPENAI_ENDPOINT")  # e.g. https://project.openai.azure.com
+    azure_key = os.getenv("AZURE_OPENAI_API_KEY")
+    azure_deployment = os.getenv("AZURE_OPENAI_DEPLOYMENT")  # i.e. deployment name with some models deployed
+    azure_api_version = os.getenv('AZURE_OPENAI_API_VERSION', '2024-07-01-preview')
+    assert azure_endpoint is not None, "Azure OpenAI endpoint not set"
+    assert azure_key is not None, "Azure OpenAI API key not set"
+    assert azure_deployment is not None, "Azure OpenAI deployment not set"
+
+    client = AzureOpenAI(
+        azure_endpoint=azure_endpoint,
+        api_key=azure_key,
+        api_version=azure_api_version,
+        azure_deployment=azure_deployment,
+    )
+
+    if chat_history is None:
+        chat_history = []
+
+    messages = [{"role": "system", "content": system}] + chat_history + [{"role": "user", "content": prompt}]
+
+    response = openai_completion_with_backoff(client,
+                                              model=model,
+                                              messages=messages,
+                                              temperature=temperature,
+                                              max_tokens=max_tokens,
+                                              stream=True
+                                              )
+
+    output_tokens = 0
+    input_tokens = 0
+    for chunk in response:
+        if chunk.choices and chunk.choices[0].delta.content:
+            yield dict(text=chunk.choices[0].delta.content)
+        if chunk.usage:
+            output_tokens = chunk.usage.completion_tokens
+            input_tokens = chunk.usage.prompt_tokens
+
+    if verbose:
+        print(f"Output tokens: {output_tokens}")
+        print(f"Input tokens: {input_tokens}")
+    yield dict(output_tokens=output_tokens, input_tokens=input_tokens)
+
+
+def to_list(x):
+    if x:
+        try:
+            ollama_model_list = ast.literal_eval(x)
+            assert isinstance(ollama_model_list, list)
+        except:
+            x = [x]
+    else:
+        x = []
+    return x
+
+
+def get_model_names(secrets, on_hf_spaces=False):
+    if not on_hf_spaces:
+        secrets = os.environ
+    if secrets.get('ANTHROPIC_API_KEY'):
+        anthropic_models = ['claude-3-5-sonnet-20240620', 'claude-3-haiku-20240307', 'claude-3-opus-20240229']
+    else:
+        anthropic_models = []
+    if secrets.get('OPENAI_API_KEY'):
+        if os.getenv('OPENAI_MODEL_NAME'):
+            openai_models = to_list(os.getenv('OPENAI_MODEL_NAME'))
+        else:
+            openai_models = ['gpt-4o', 'gpt-4-turbo-2024-04-09', 'gpt-4o-mini']
+    else:
+        openai_models = []
+    if secrets.get('AZURE_OPENAI_API_KEY'):
+        if os.getenv('AZURE_OPENAI_MODEL_NAME'):
+            azure_models = to_list(os.getenv('AZURE_OPENAI_MODEL_NAME'))
+        else:
+            azure_models = ['gpt-4o', 'gpt-4-turbo-2024-04-09', 'gpt-4o-mini']
+    else:
+        azure_models = []
+    if secrets.get('GEMINI_API_KEY'):
+        google_models = ['gemini-1.5-pro-latest', 'gemini-1.5-flash-latest']
+    else:
+        google_models = []
+    if secrets.get('GROQ_API_KEY'):
+        groq_models = ['llama-3.1-70b-versatile',
+                       'llama-3.1-8b-instant',
+                       'llama3-groq-70b-8192-tool-use-preview',
+                       'llama3-groq-8b-8192-tool-use-preview',
+                       'mixtral-8x7b-32768']
+    else:
+        groq_models = []
+    if secrets.get('OLLAMA_OPENAI_API_KEY'):
+        ollama_model = os.environ['OLLAMA_OPENAI_MODEL_NAME']
+        ollama_model = to_list(ollama_model)
+    else:
+        ollama_model = []
+
+    groq_models = ['groq:' + x for x in groq_models]
+    azure_models = ['azure:' + x for x in azure_models]
+    openai_models = ['openai:' + x for x in openai_models]
+    google_models = ['google:' + x for x in google_models]
+    anthropic_models = ['anthropic:' + x for x in anthropic_models]
+    ollama = ['ollama:' + x if 'ollama:' not in x else x for x in ollama_model]
+
+    return anthropic_models, openai_models, google_models, groq_models, azure_models, ollama
+
+
+def get_all_model_names(secrets, on_hf_spaces=False):
+    anthropic_models, openai_models, google_models, groq_models, azure_models, ollama = get_model_names(secrets,
+                                                                                                        on_hf_spaces=on_hf_spaces)
+    return anthropic_models + openai_models + google_models + groq_models + azure_models + ollama
+
+
+def get_model_api(model: str):
+    if model.startswith('anthropic:'):
+        return get_anthropic
+    elif model.startswith('openai:') or model.startswith('ollama:'):
+        return get_openai
+    elif model.startswith('google:'):
+        return get_google
+    elif model.startswith('groq:'):
+        return get_groq
+    elif model.startswith('azure:'):
+        return get_openai_azure
+    else:
+        raise ValueError(
+            f"Unsupported model: {model}.  Ensure to add prefix (e.g. openai:, google:, groq:, azure:, ollama:, anthropic:)")

open_strawberry.py

@@ -0,0 +1,491 @@
+import os
+import random
+import time
+from typing import List, Dict, Generator, Tuple
+from collections import deque
+
+try:
+    from src.models import get_model_api
+    from src.utils import get_turn_title, get_final_answer, get_xml_tag_value
+except (ModuleNotFoundError, ImportError):
+    from models import get_model_api
+    from utils import get_turn_title, get_final_answer, get_xml_tag_value
+
+
+class DeductionTracker:
+    def __init__(self):
+        self.deductions = []
+        self.certainty_scores = []
+
+    def add_deduction(self, deduction: str, certainty: float):
+        self.deductions.append(deduction)
+        self.certainty_scores.append(certainty)
+
+    def get_deductions(self):
+        return list(zip(self.deductions, self.certainty_scores))
+
+    def update_certainty(self, index: int, new_certainty: float):
+        if 0 <= index < len(self.certainty_scores):
+            self.certainty_scores[index] = new_certainty
+
+
+class ProblemRepresentation:
+    def __init__(self):
+        self.current_representation = ""
+
+    def update(self, new_representation: str):
+        self.current_representation = new_representation
+
+    def get(self) -> str:
+        return self.current_representation
+
+
+def get_last_assistant_responses(chat_history, n=3):
+    assistant_messages = [msg['content'] for msg in chat_history if msg['role'] == 'assistant']
+    return assistant_messages[-n:]
+
+
+def generate_dynamic_system_prompt(base_prompt, turn_count: int, problem_complexity: float, problem_representation: str,
+                                   deductions: List[Tuple[str, float]]) -> str:
+    dynamic_prompt = base_prompt + "\n\n* Always refer to and update the current problem representation as needed."
+    dynamic_prompt += "\n* Maintain and update the list of deductions and their certainty scores."
+
+    if turn_count > 20:
+        dynamic_prompt += "\n* At this stage, focus on synthesizing your previous thoughts and looking for breakthrough insights."
+
+    if problem_complexity > 0.7:
+        dynamic_prompt += "\n* This is a highly complex problem. Consider breaking it down into smaller subproblems and solving them incrementally."
+
+    dynamic_prompt += "\n* Regularly verify that your current understanding satisfies ALL given clues."
+    dynamic_prompt += "\n* If you reach a contradiction, backtrack to the last point where you were certain and explore alternative paths."
+
+    dynamic_prompt += f"\n\nCurrent problem representation:\n{problem_representation}"
+    dynamic_prompt += "\n\nYou can update this representation by providing a new one within <representation></representation> tags."
+
+    dynamic_prompt += "\n\nCurrent deductions and certainty scores:"
+    for deduction, certainty in deductions:
+        dynamic_prompt += f"\n- {deduction} (Certainty: {certainty})"
+    dynamic_prompt += "\n\nYou can add new deductions or update existing ones using <deduction></deduction> and <certainty></certainty> tags."
+
+    return dynamic_prompt
+
+
+def generate_initial_representation_prompt(initial_prompt: str) -> str:
+    return f"""Based on the following problem description:
+{initial_prompt}
+
+Representation:
+* Create a clear and clean representation that breaks down the problem into parts in order to create a structure that helps track solving the problem.
+* Put the representation inside <representation> </representation> XML tags, ensuring to add new lines before and after XML tags.
+* The representation could be a table, matrix, grid, or any other format that breaks down the problem into its components and ensure it helps iteratively track progress towards the solution.
+* Example representations include a matrix that has values of digits as rows and position of digits as columns and values at each row-column as tracking confirmed position, eliminated position, or possible position.
+* For a table or grid representation, you must put the table or grid inside a Markdown code block (with new lines around the back ticks) and make it nice and easy to read for a human to understand it.
+
+Deductions:
+* Provide your initial deductions (if any) using <deduction></deduction> tags, each followed by a certainty score in <certainty></certainty> tags (0-100).
+"""
+
+
+def generate_verification_prompt(chat_history: List[Dict], turn_count: int, problem_representation: str,
+                                 deductions: List[Tuple[str, float]]) -> str:
+    last_responses = get_last_assistant_responses(chat_history, n=5)
+
+    verification_prompt = f"""Turn {turn_count}: Comprehensive Verification and Critique
+
+1. Review your previous reasoning steps:
+{' '.join(last_responses)}
+
+2. Current problem representation:
+{problem_representation}
+
+3. Current deductions and certainty scores:
+{deductions}
+
+4. Perform the following checks:
+   a) Identify any logical fallacies or unjustified assumptions
+   b) Check for mathematical or factual errors
+   c) Assess the relevance of each step to the main problem
+   d) Evaluate the coherence and consistency of your reasoning
+   e) Verify that your current understanding satisfies ALL given clues
+   f) Check if any of your deductions contradict each other
+
+5. If you find any issues:
+   a) Explain the issue in detail
+   b) Correct the error or resolve the contradiction
+   c) Update the problem representation if necessary
+   d) Update deductions and certainty scores as needed
+
+6. If no issues are found, suggest a new approach or perspective to consider.
+
+7. Assign an overall confidence score (0-100) to your current reasoning path and explain why.
+
+Respond in this format:
+<verification>
+[Your detailed verification and critique]
+</verification>
+<updates>
+[Any updates to the problem representation or deductions]
+</updates>
+<confidence_score>[0-100]</confidence_score>
+<explanation>[Explanation for the confidence score]</explanation>
+
+If you need to update the problem representation, provide the new representation within <representation></representation> tags.
+For new or updated deductions, use <deduction></deduction> tags, each followed by <certainty></certainty> tags.
+"""
+    return verification_prompt
+
+
+def generate_hypothesis_prompt(chat_history: List[Dict]) -> str:
+    return """Based on your current understanding of the problem:
+1. Generate three distinct hypotheses that could lead to a solution.
+2. For each hypothesis, provide a brief rationale and a potential test to validate it.
+3. Rank these hypotheses in order of perceived promise.
+
+Respond in this format:
+<hypotheses>
+1. [Hypothesis 1]
+   Rationale: [Brief explanation]
+   Test: [Proposed validation method]
+
+2. [Hypothesis 2]
+   Rationale: [Brief explanation]
+   Test: [Proposed validation method]
+
+3. [Hypothesis 3]
+   Rationale: [Brief explanation]
+   Test: [Proposed validation method]
+</hypotheses>
+<ranking>[Your ranking and brief justification]</ranking>
+"""
+
+
+def generate_analogical_reasoning_prompt(problem_description: str) -> str:
+    return f"""Consider the following problem:
+{problem_description}
+
+Now, think of an analogous problem from a different domain that shares similar structural characteristics. Describe:
+1. The analogous problem
+2. The key similarities between the original and analogous problems
+3. How the solution to the analogous problem might inform our approach to the original problem
+
+Respond in this format:
+<analogy>
+Problem: [Description of the analogous problem]
+Similarities: [Key structural similarities]
+Insights: [How this analogy might help solve the original problem]
+</analogy>
+"""
+
+
+def generate_metacognitive_prompt() -> str:
+    return """Take a step back and reflect on your problem-solving process:
+1. What strategies have been most effective so far?
+2. What are the main obstacles you're facing?
+3. How might you adjust your approach to overcome these obstacles?
+4. What assumptions might you be making that could be limiting your progress?
+
+Respond in this format:
+<metacognition>
+Effective Strategies: [List and brief explanation]
+Main Obstacles: [List and brief explanation]
+Proposed Adjustments: [List of potential changes to your approach]
+Potential Limiting Assumptions: [List and brief explanation]
+</metacognition>
+"""
+
+
+def generate_devils_advocate_prompt(current_approach: str) -> str:
+    return f"""Consider your current approach:
+{current_approach}
+
+Now, play the role of a skeptical critic:
+1. What are the three strongest arguments against this approach?
+2. What critical information might we be overlooking?
+3. How might this approach fail in extreme or edge cases?
+
+Respond in this format:
+<devils_advocate>
+Counter-arguments:
+1. [First strong counter-argument]
+2. [Second strong counter-argument]
+3. [Third strong counter-argument]
+
+Overlooked Information: [Potential critical information we might be missing]
+
+Potential Failures: [How this approach might fail in extreme or edge cases]
+</devils_advocate>
+"""
+
+
+def generate_hint(problem_description: str, current_progress: str, difficulty: float) -> str:
+    if difficulty < 0.3:
+        hint_level = "subtle"
+    elif difficulty < 0.7:
+        hint_level = "moderate"
+    else:
+        hint_level = "strong"
+
+    return f"""Based on the original problem:
+{problem_description}
+
+And the current progress:
+{current_progress}
+
+Provide a {hint_level} hint to help move towards the solution without giving it away entirely.
+
+<hint>
+[Your {hint_level} hint here]
+</hint>
+"""
+
+
+def summarize_and_restructure(chat_history: List[Dict]) -> str:
+    return """Review the entire conversation history and provide:
+1. A concise summary of the key insights and progress made so far
+2. A restructured presentation of the problem based on our current understanding
+3. Identification of any patterns or recurring themes in our problem-solving attempts
+
+Respond in this format:
+<summary_and_restructure>
+Key Insights: [Bullet point list of main insights]
+Restructured Problem: [Revised problem statement based on current understanding]
+Patterns/Themes: [Identified patterns or recurring themes in our approach]
+</summary_and_restructure>
+"""
+
+
+class Memory:
+    def __init__(self, max_size=10):
+        self.insights = deque(maxlen=max_size)
+        self.mistakes = deque(maxlen=max_size)
+        self.dead_ends = deque(maxlen=max_size)
+
+    def add_insight(self, insight: str):
+        self.insights.append(insight)
+
+    def add_mistake(self, mistake: str):
+        self.mistakes.append(mistake)
+
+    def add_dead_end(self, dead_end: str):
+        self.dead_ends.append(dead_end)
+
+    def get_insights(self) -> List[str]:
+        return list(self.insights)
+
+    def get_mistakes(self) -> List[str]:
+        return list(self.mistakes)
+
+    def get_dead_ends(self) -> List[str]:
+        return list(self.dead_ends)
+
+
+def manage_conversation(model: str,
+                        system: str,
+                        initial_prompt: str,
+                        next_prompts: List[str],
+                        final_prompt: str = "",
+                        num_turns: int = 25,
+                        num_turns_final_mod: int = 9,
+                        cli_mode: bool = False,
+                        temperature: float = 0.3,
+                        max_tokens: int = 4096,
+                        seed: int = 1234,
+                        verbose: bool = False
+                        ) -> Generator[Dict, None, list]:
+    if seed == 0:
+        seed = random.randint(0, 1000000)
+    random.seed(seed)
+    get_model_func = get_model_api(model)
+    chat_history = []
+    memory = Memory()
+    problem_representation = ProblemRepresentation()
+    deduction_tracker = DeductionTracker()
+
+    turn_count = 0
+    total_thinking_time = 0
+    problem_complexity = 0.5  # Initial estimate, will be dynamically updated
+
+    base_system = system
+    while True:
+        system = generate_dynamic_system_prompt(base_system, turn_count, problem_complexity,
+                                                problem_representation.get(), deduction_tracker.get_deductions())
+        trying_final = False
+
+        if turn_count == 0:
+            prompt = generate_initial_representation_prompt(initial_prompt)
+        elif turn_count % 5 == 0:
+            prompt = generate_verification_prompt(chat_history, turn_count, problem_representation.get(),
+                                                  deduction_tracker.get_deductions())
+        elif turn_count % 7 == 0:
+            prompt = generate_hypothesis_prompt(chat_history)
+        elif turn_count % 11 == 0:
+            prompt = generate_analogical_reasoning_prompt(initial_prompt)
+        elif turn_count % 13 == 0:
+            prompt = generate_metacognitive_prompt()
+        elif turn_count % 17 == 0:
+            current_approach = get_last_assistant_responses(chat_history, n=1)[0]
+            prompt = generate_devils_advocate_prompt(current_approach)
+        elif turn_count % 19 == 0:
+            current_progress = get_last_assistant_responses(chat_history, n=3)
+            prompt = generate_hint(initial_prompt, "\n".join(current_progress), problem_complexity)
+        elif turn_count % 23 == 0:
+            prompt = summarize_and_restructure(chat_history)
+        elif turn_count % num_turns_final_mod == 0 and turn_count > 0:
+            trying_final = True
+            prompt = final_prompt
+        else:
+            prompt = random.choice(next_prompts)
+
+        if turn_count == 0:
+            yield {"role": "user", "content": initial_prompt, "chat_history": chat_history, "initial": turn_count == 0}
+        else:
+            yield {"role": "user", "content": prompt, "chat_history": chat_history, "initial": turn_count == 0}
+
+        thinking_time = time.time()
+        response_text = ''
+        for chunk in get_model_func(model, prompt, system=system, chat_history=chat_history,
+                                    temperature=temperature, max_tokens=max_tokens, verbose=verbose):
+            if 'text' in chunk and chunk['text']:
+                response_text += chunk['text']
+                yield {"role": "assistant", "content": chunk['text'], "streaming": True, "chat_history": chat_history,
+                       "final": False, "turn_title": False}
+            else:
+                yield {"role": "usage", "content": chunk}
+        thinking_time = time.time() - thinking_time
+        total_thinking_time += thinking_time
+
+        # Update problem complexity based on thinking time and response length
+        problem_complexity = min(1.0,
+                                 problem_complexity + (thinking_time / 60) * 0.1 + (len(response_text) / 1000) * 0.05)
+
+        # Extract and update problem representation
+        representations = get_xml_tag_value(response_text, 'representation', ret_all=False)
+        if representations:
+            problem_representation.update(representations[-1])
+
+        # Extract and update deductions
+        deductions = get_xml_tag_value(response_text, 'deduction')
+        for deduction in deductions:
+            certainties = get_xml_tag_value(response_text, 'certainty', ret_all=False)
+            if certainties:
+                deduction_tracker.add_deduction(deduction, certainties[-1])
+
+        # Extract insights, mistakes, and dead ends from the response
+        [memory.add_insight(x) for x in get_xml_tag_value(response_text, 'insight')]
+        [memory.add_mistake(x) for x in get_xml_tag_value(response_text, 'mistake')]
+        [memory.add_dead_end(x) for x in get_xml_tag_value(response_text, 'dead_end')]
+
+        turn_title = get_turn_title(response_text)
+        yield {"role": "assistant", "content": turn_title, "turn_title": True, 'thinking_time': thinking_time,
+               'total_thinking_time': total_thinking_time}
+
+        chat_history.append(
+            {"role": "user",
+             "content": [{"type": "text", "text": prompt, "cache_control": {"type": "ephemeral"}}]})
+        chat_history.append({"role": "assistant", "content": response_text})
+
+        # Adjusted to only check final answer when trying_final is True
+        always_check_final = False
+        if trying_final or always_check_final:
+            final_value = get_final_answer(response_text, cli_mode=cli_mode)
+            if final_value:
+                chat_history.append({"role": "assistant", "content": final_value})
+                yield {"role": "assistant", "content": final_value, "streaming": True, "chat_history": chat_history,
+                       "final": True}
+                break
+
+        turn_count += 1
+
+        # Dynamically adjust temperature based on progress
+        if turn_count % 10 == 0:
+            temperature = min(1.0, temperature + 0.1)  # Gradually increase temperature to encourage exploration
+
+        if turn_count % num_turns == 0:
+            # periodically pause for continuation, never have to fully terminate
+            if cli_mode:
+                user_continue = input("\nContinue? (y/n): ").lower() == 'y'
+                if not user_continue:
+                    break
+            else:
+                yield {"role": "action", "content": "continue?", "chat_history": chat_history}
+
+        time.sleep(0.001)
+
+
+def get_defaults() -> Tuple:
+    initial_prompt = """Can you crack the code?
+9 2 8 5 (One number is correct but in the wrong position)
+1 9 3 7 (Two numbers are correct but in the wrong positions)
+5 2 0 1 (one number is correct and in the right position)
+6 5 0 7 (nothing is correct)
+8 5 2 4 (two numbers are correct but in the wrong positions)"""
+
+    expected_answer = "3841"
+
+    system_prompt = """Let us play a game of "take only the most minuscule step toward the solution."
+<thinking_game>
+* The assistant's text output must be only the very next possible step.
+* Use your text output as a scratch pad in addition to a literal output of some next step.
+* Every time you make a major shift in thinking, output your high-level current thinking in <thinking> </thinking> XML tags.
+* You should present your response in a way that iterates on that scratch pad space with surrounding textual context.
+* You win the game if you are able to take the smallest text steps possible while still (on average) heading towards the solution.
+* Backtracking is allowed, and generating python code is allowed (but will not be executed, but can be used to think), just on average over many text output turns you must head towards the answer.
+* You must think using first principles, and ensure you identify inconsistencies, errors, etc.
+* Periodically, you should review your previous reasoning steps and check for errors or inconsistencies. If you find any, correct them.
+* You MUST always end with a very brief natural language title (it should just describe the analysis, do not give step numbers) of what you did inside <turn_title> </turn_title> XML tags. Only a single title is allowed.
+* Do not provide the final answer unless the user specifically requests it using the final prompt.
+</thinking_game>
+Remember to compensate for your flaws:
+<system_flaws>
+* Flaw 1: Bad at counting due to tokenization issues.  Expand word with spaces between first and then only count that expanded version.
+* Flaw 2: Grade school or advanced math.  Solve such problems very carefully step-by-step.
+</system_flaws>
+"""
+
+    next_prompts = [
+        "Continue your effort to answer the original query. What's your next step?",
+        "What aspect of the problem haven't we considered yet?",
+        "Can you identify any patterns or relationships in the given information?",
+        "How would you verify your current reasoning?",
+        "What's the weakest part of your current approach? How can we strengthen it?",
+        "If you were to explain your current thinking to a novice, what would you say?",
+        "What alternative perspectives could we consider?",
+        "How does your current approach align with the constraints of the problem?",
+        "What assumptions are we making? Are they all necessary?",
+        "If we were to start over with our current knowledge, how would our approach differ?",
+    ]
+
+    final_prompt = """Verification checklist:
+1) Do you have very high confidence in a final answer?
+2) Have you fully verified your answer with all the time and resources you have?
+3) If you have very high confidence AND you have fully verified your answer with all resources possible, then put the final answer in <final_answer> </final_answer> XML tags, otherwise please continue to vigorously work on the user's original query.
+"""
+
+    num_turns = int(os.getenv('NUM_TURNS', '10'))  # Number of turns before pausing for continuation
+    num_turns_final_mod = num_turns - 1  # Not required, just an OK value. Could be randomized.
+
+    show_next = False
+    show_cot = False
+    verbose = False
+
+    # model = "claude-3-5-sonnet-20240620"
+    model = "anthropic:claude-3-haiku-20240307"
+
+    temperature = 0.3
+    max_tokens = 4096
+
+    return (model, system_prompt,
+            initial_prompt,
+            expected_answer,
+            next_prompts,
+            num_turns, show_next, final_prompt,
+            temperature, max_tokens,
+            num_turns_final_mod,
+            show_cot,
+            verbose)
+
+
+if __name__ == '__main__':
+    from src.cli import go_cli
+
+    go_cli()

requirements.txt

@@ -0,0 +1,13 @@
+python-dotenv
+
+anthropic
+openai
+google-generativeai
+groq
+
+streamlit
+blinker
+click
+pydantic
+
+tenacity

utils.py

@@ -0,0 +1,57 @@
+import re
+
+
+def get_turn_title(response_text):
+    tag = 'turn_title'
+    pattern = fr'<{tag}>(.*?)</{tag}>'
+    values = re.findall(pattern, response_text, re.DOTALL)
+    values0 = values.copy()
+    values = [v.strip() for v in values]
+    values = [v for v in values if v]
+    if len(values) == 0:
+        # then maybe removed too much
+        values = values0
+    if values:
+        turn_title = values[-1]
+    else:
+        turn_title = response_text[:15] + '...'
+    return turn_title
+
+
+def get_final_answer(response_text, cli_mode=False):
+    tag = 'final_answer'
+    pattern = fr'<{tag}>(.*?)</{tag}>'
+    values = re.findall(pattern, response_text, re.DOTALL)
+    values0 = values.copy()
+    values = [v.strip() for v in values]
+    values = [v for v in values if v]
+    if len(values) == 0:
+        # then maybe removed too much
+        values = values0
+    if values:
+        if cli_mode:
+            response_text = '\n\nFINAL ANSWER:\n\n' + values[-1] + '\n\n'
+        else:
+            response_text = values[-1]
+    else:
+        response_text = None
+    return response_text
+
+
+def get_xml_tag_value(response_text, tag, ret_all=True):
+    pattern = fr'<{tag}>(.*?)</{tag}>'
+    values = re.findall(pattern, response_text, re.DOTALL)
+    values0 = values.copy()
+    values = [v.strip() for v in values]
+    values = [v for v in values if v]
+    if len(values) == 0:
+        # then maybe removed too much
+        values = values0
+    if values:
+        if ret_all:
+            ret = values
+        else:
+            ret = [values[-1]]
+    else:
+        ret = []
+    return ret