app.py

!pip install torch==2.0.1 transformers==4.27.1 datasets==2.4.0 wget==3.2 huggingface-hub==0.14.1 beautifulsoup4==4.11.1 requests==2.28.1 matplotlib tqdm python-dotenv diffusers

import os
import torch
import torch.nn as nn
from torch.utils.data import DataLoader, Dataset
from torch.optim import AdamW
import matplotlib.pyplot as plt
import matplotlib.animation as animation
import time
import threading
from tqdm import tqdm
from transformers import AutoTokenizer, AutoModel, TrainingArguments, pipeline
from diffusers import DiffusionPipeline
from huggingface_hub import login, HfApi, Repository
from dotenv import load_dotenv

# Cargar variables de entorno
load_dotenv()

class UnifiedModel(nn.Module):
    def __init__(self, models):
        super(UnifiedModel, self).__init__()
        self.models = nn.ModuleList(models)
        self.classifier = nn.Linear(sum([model.config.hidden_size for model in models if hasattr(model, 'config')]), 2)

    def forward(self, inputs):
        hidden_states = []
        for model in self.models:
            if isinstance(model, nn.Module):
                outputs = model(inputs)
                hidden_states.append(outputs.last_hidden_state[:, 0, :])
            elif isinstance(model, DiffusionPipeline) or isinstance(model, pipeline):
                outputs = model(inputs)
                hidden_states.append(torch.tensor(outputs))
        concatenated_hidden_states = torch.cat(hidden_states, dim=-1)
        logits = self.classifier(concatenated_hidden_states)
        return logits


class SyntheticDataset(Dataset):
    def __init__(self, tokenizers, size=100):
        self.tokenizers = tokenizers
        self.size = size
        self.data = self._generate_data()

    def _generate_data(self):
        data = []
        for _ in range(self.size):
            text = "This is a sample sentence for testing purposes."
            label = torch.tensor(0)  # Sample label
            item = {"text": text, "label": label}
            for name, tokenizer in self.tokenizers.items():
                tokenized = tokenizer(text, padding="max_length", truncation=True, max_length=128)
                item[f"input_ids_{name}"] = torch.tensor(tokenized["input_ids"])
                item[f"attention_mask_{name}"] = torch.tensor(tokenized["attention_mask"])
            data.append(item)
        return data

    def __len__(self):
        return len(self.data)

    def __getitem__(self, idx):
        return self.data[idx]


def push_to_hub(local_dir, repo_name):
    try:
        repo_url = HfApi().create_repo(repo_name, exist_ok=True)
        repo = Repository(local_dir, clone_from=repo_url)

        if not os.path.exists(os.path.join(local_dir, ".git")):
            os.system(f"cd {local_dir} && git init && git remote add origin {repo_url} && git pull origin main")

        repo.git_add(auto_lfs_track=True)
        repo.git_commit("Add model and tokenizer files")

        json_files = ["config.json", "generation_config.json", "special_tokens_map.json", "tokenizer.json", "tokenizer.model", "tokenizer_config.json"]
        for json_file in json_files:
            json_file_path = os.path.join(local_dir, json_file)
            if os.path.exists(json_file_path):
                repo.git_add(json_file_path)

        repo.git_push()
        print(f"Pushed model and tokenizer to {repo_url}")
    except Exception as e:
        print(f"Error pushing to Hugging Face Hub: {e}")


def main():
    while True:
        try:
            os.system("git config --global credential.helper store")
            login(token=os.getenv("HUGGINGFACE_TOKEN"), add_to_git_credential=True)

            # Definir los modelos que se van a utilizar
            models_to_train = [
                "openai-community/gpt2-xl",
                "google/gemma-2-9b-it",
                "google/gemma-2-9b",
                "meta-llama/Meta-Llama-3.1-8B-Instruct",
                "meta-llama/Meta-Llama-3.1-8B",
                "openbmb/MiniCPM-V-2_6",
                "bigcode/starcoder",
                "WizardLMTeam/WizardCoder-Python-34B-V1.0",
                "Qwen/Qwen2-72B-Instruct",
                "google/gemma-2-2b-it",
                "facebook/bart-large-cnn",
                "Falconsai/text_summarization",
                "microsoft/speecht5_tts",
                "Groq/Llama-3-Groq-70B-Tool-Use",
                "Groq/Llama-3-Groq-8B-Tool-Use"
            ]

            # Inicializar los pipelines
            pipelines_to_unify = [
                pipeline("text-to-audio", model="facebook/musicgen-melody"),
                pipeline("text-to-audio", model="facebook/musicgen-large"),
                pipeline("text-to-audio", model="facebook/musicgen-small"),
                DiffusionPipeline.from_pretrained("stabilityai/stable-video-diffusion-img2vid-xt-1-1"),
                pipeline("automatic-speech-recognition", model="openai/whisper-small"),
                DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-dev"),
                DiffusionPipeline.from_pretrained("stabilityai/stable-diffusion-2-1"),
                DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-schnell"),
                pipeline("text-generation", model="meta-llama/Meta-Llama-3.1-8B"),
                pipeline("text-generation", model="openbmb/MiniCPM-V-2_6"),
                pipeline("text-generation", model="bigcode/starcoder"),
                pipeline("text-to-speech", model="microsoft/speecht5_tts"),
                pipeline("text-generation", model="WizardLMTeam/WizardCoder-Python-34B-V1.0"),
                pipeline("text-generation", model="Qwen/Qwen2-72B-Instruct"),
                pipeline("text-generation", model="google/gemma-2-2b-it"),
                pipeline("summarization", model="facebook/bart-large-cnn"),
                pipeline("summarization", model="Falconsai/text_summarization"),
                DiffusionPipeline.from_pretrained("black-forest-labs/FLUX.1-dev"),
                pipeline("text-to-audio", model="facebook/musicgen-small"),
                pipeline("text-generation", model="Groq/Llama-3-Groq-70B-Tool-Use"),
                pipeline("text-generation", model="Groq/Llama-3-Groq-8B-Tool-Use")
            ]

            tokenizers = {}
            models = []
            for model_name in models_to_train:
                tokenizer = AutoTokenizer.from_pretrained(model_name, use_fast=True)

                if tokenizer.pad_token is None:
                    tokenizer.add_special_tokens({'pad_token': tokenizer.eos_token})

                model = AutoModel.from_pretrained(model_name)
                tokenizers[model_name] = tokenizer
                models.append(model)

            # Agregar pipelines como modelos
            models.extend(pipelines_to_unify)

            # Crear un dataset sintético para entrenamiento y evaluación
            synthetic_dataset = SyntheticDataset(tokenizers, size=100)

            # Dividir el dataset en entrenamiento y evaluación
            train_size = int(0.8 * len(synthetic_dataset))
            val_size = len(synthetic_dataset) - train_size
            train_dataset, val_dataset = torch.utils.data.random_split(synthetic_dataset, [train_size, val_size])

            # Crear DataLoaders para entrenamiento y evaluación
            train_loader = DataLoader(train_dataset, batch_size=2, shuffle=True)
            eval_loader = DataLoader(val_dataset, batch_size=16)

            # Unificar los modelos y pipelines en uno solo
            unified_model = UnifiedModel(models)
            unified_model.to(torch.device("cpu"))

            # Mostrar la cantidad de parámetros totales a entrenar
            total_params = sum(p.numel() for p in unified_model.parameters())
            print(f"Total parameters to train: {total_params}")

            # Definir los argumentos de entrenamiento
            training_args = TrainingArguments(
                output_dir="outputs/unified_model",
                evaluation_strategy="epoch",
                learning_rate=9e-4,
                per_device_train_batch_size=2,
                per_device_eval_batch_size=16,
                num_train_epochs=1,  # Reduced epochs for quick training
                weight_decay=0.01,
                logging_steps=10,  # More frequent logging for quicker feedback
                optim="adamw_hf"
            )

            # Definir el optimizador
            optimizer = AdamW(unified_model.parameters(), lr=training_args.learning_rate)

            train_losses = []
            eval_losses = []

            def train(model, train_loader, eval_loader, args):
                model.train()
                epoch = 0
                total_steps = args.num_train_epochs * len(train_loader)
                progress_bar = tqdm(total=total_steps, desc="Training")

                while epoch < args.num_train_epochs:
                    start_time = time.time()
                    for step, batch in enumerate(train_loader):
                        input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
                        attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
                        labels = batch["label"].to("cpu")
                        optimizer.zero_grad()
                        outputs = model(input_ids)
                        loss = nn.CrossEntropyLoss()(outputs, labels)
                        loss.backward()
                        optimizer.step()
                        progress_bar.update(1)

                        elapsed_time = time.time() - start_time
                        estimated_total_time = total_steps * (elapsed_time / (step + 1))
                        estimated_remaining_time = estimated_total_time - elapsed_time

                        if step % args.logging_steps == 0:
                            train_losses.append(loss.item())
                            print(f"Step {step}/{total_steps}, Loss: {loss.item()}, Estimated remaining time: {estimated_remaining_time:.2f} seconds")

                    epoch += 1
                    model.eval()
                    eval_loss = 0
                    with torch.no_grad():
                        for batch in eval_loader:
                            input_ids = [batch[f"input_ids_{name}"].to("cpu") for name in tokenizers.keys()]
                            attention_mask = [batch[f"attention_mask_{name}"].to("cpu") for name in tokenizers.keys()]
                            labels = batch["label"].to("cpu")
                            outputs = model(input_ids)
                            loss = nn.CrossEntropyLoss()(outputs, labels)
                            eval_loss += loss.item()

                    eval_loss /= len(eval_loader)
                    eval_losses.append(eval_loss)
                    print(f"Epoch {epoch}/{args.num_train_epochs}, Evaluation Loss: {eval_loss}")

            train(unified_model, train_loader, eval_loader, training_args)

            # Visualizar pérdidas durante el entrenamiento
            fig, ax = plt.subplots()
            ax.set_xlabel("Epochs")
            ax.set_ylabel("Loss")
            ax.legend()

            def animate(i):
                ax.clear()
                ax.plot(train_losses[:i], label="Train Loss")
                ax.plot(eval_losses[:i], label="Eval Loss")
                ax.legend()

            ani = animation.FuncAnimation(fig, animate, frames=len(train_losses), blit=False)
            plt.show()

            # Subir el modelo unificado a Hugging Face Hub
            local_dir = "./outputs/unified_model"
            push_to_hub(local_dir, repo_name="Ffftdtd5dtft/my_model")

            break
        except Exception as e:
            print(f"Error: {e}")
            time.sleep(2)

if __name__ == "__main__":
    main()