Spaces:
Runtime error
Runtime error
File size: 4,736 Bytes
e3a7b6f 1608585 e3a7b6f e6dda1e e3a7b6f 1608585 e3a7b6f 5e2b717 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 1608585 e3a7b6f 5e2b717 e3a7b6f e6dda1e e3a7b6f e6dda1e e3a7b6f e6dda1e e3a7b6f e6dda1e e3a7b6f |
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 |
from fastapi import FastAPI, HTTPException
from pydantic import BaseModel
from llama_cpp import Llama
from concurrent.futures import ProcessPoolExecutor, as_completed
import uvicorn
from dotenv import load_dotenv
from difflib import SequenceMatcher
from tqdm import tqdm
import threading
load_dotenv()
app = FastAPI()
# Configuraci贸n de los modelos
models = [
{"repo_id": "Ffftdtd5dtft/gpt2-xl-Q2_K-GGUF", "filename": "gpt2-xl-q2_k.gguf"},
{"repo_id": "Ffftdtd5dtft/Meta-Llama-3.1-8B-Instruct-Q2_K-GGUF", "filename": "meta-llama-3.1-8b-instruct-q2_k.gguf"},
{"repo_id": "Ffftdtd5dtft/gemma-2-9b-it-Q2_K-GGUF", "filename": "gemma-2-9b-it-q2_k.gguf"},
{"repo_id": "Ffftdtd5dtft/gemma-2-27b-Q2_K-GGUF", "filename": "gemma-2-27b-q2_k.gguf"},
]
# Funci贸n para cargar un modelo
def load_model(model_config):
return Llama.from_pretrained(repo_id=model_config['repo_id'], filename=model_config['filename'])
# Cargar modelos en paralelo
def load_all_models():
with ProcessPoolExecutor() as executor:
future_to_model = {executor.submit(load_model, model): model for model in models}
loaded_models = {}
for future in as_completed(future_to_model):
model = future_to_model[future]
try:
loaded_models[model['repo_id']] = future.result()
print(f"Modelo cargado en RAM: {model['repo_id']}")
except Exception as exc:
print(f"Error al cargar modelo {model['repo_id']}: {exc}")
return loaded_models
# Cargar modelos en memoria
llms = load_all_models()
class ChatRequest(BaseModel):
message: str
top_k: int = 50
top_p: float = 0.95
temperature: float = 0.7
# Funci贸n global para generar respuestas de chat
def generate_chat_response(request, llm):
try:
user_input = normalize_input(request.message)
response = llm.create_chat_completion(
messages=[{"role": "user", "content": user_input}],
top_k=request.top_k,
top_p=request.top_p,
temperature=request.temperature
)
reply = response['choices'][0]['message']['content']
return {"response": reply, "literal": user_input}
except Exception as e:
return {"response": f"Error: {str(e)}", "literal": user_input}
def normalize_input(input_text):
return input_text.strip()
def filter_duplicates(responses):
seen = set()
unique_responses = []
for response in responses:
lines = response.split('\n')
unique_lines = set()
for line in lines:
if line not in seen:
seen.add(line)
unique_lines.add(line)
unique_responses.append('\n'.join(unique_lines))
return unique_responses
def select_best_response(responses):
# Eliminar respuestas duplicadas
unique_responses = filter_duplicates(responses)
# Deduplicar respuestas
unique_responses = list(set(unique_responses))
# Filtrar respuestas coherentes
coherent_responses = filter_by_coherence(unique_responses)
# Seleccionar la mejor respuesta
best_response = filter_by_similarity(coherent_responses)
return best_response
def filter_by_coherence(responses):
# Implementa aqu铆 un filtro de coherencia si es necesario
return responses
def filter_by_similarity(responses):
responses.sort(key=len, reverse=True)
best_response = responses[0]
for i in range(1, len(responses)):
ratio = SequenceMatcher(None, best_response, responses[i]).ratio()
if ratio < 0.9:
best_response = responses[i]
break
return best_response
def worker_function(llm, request):
return generate_chat_response(request, llm)
@app.post("/generate_chat")
async def generate_chat(request: ChatRequest):
if not request.message.strip():
raise HTTPException(status_code=400, detail="The message cannot be empty.")
print(f"Procesando solicitud: {request.message}")
responses = []
threads = []
# Crear un hilo para cada modelo
for llm in llms.values():
thread = threading.Thread(target=lambda: responses.append(worker_function(llm, request)))
threads.append(thread)
thread.start()
# Esperar a que todos los hilos terminen
for thread in threads:
thread.join()
# Seleccionar la mejor respuesta
best_response = select_best_response([response['response'] for response in responses])
print(f"Mejor respuesta seleccionada: {best_response}")
return {
"best_response": best_response,
"all_responses": [response['response'] for response in responses]
}
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=7860)
|