mainscript.py

import torch
import torch.nn as nn
import torch.optim as optim
import torch.nn.init as init
from torch.utils.data import Dataset, DataLoader
import pandas as pd
import numpy as np
from sklearn.preprocessing import StandardScaler
from sklearn.model_selection import train_test_split
import os
from datetime import datetime, timedelta
import argparse
import json
import matplotlib.pyplot as plt

# Vérifier si MPS est disponible
device = torch.device("mps" if torch.backends.mps.is_available() else "cpu")
print(f"Utilisation de l'appareil: {device}")

def load_brent_data(file_path):
    print(f"Chargement des données Brent depuis {file_path}")
    brent_data = pd.read_csv(file_path)
    brent_data['brent_date'] = pd.to_datetime(brent_data['brent_date'])
    
    # Filtrer les données à partir de 2024
    brent_data = brent_data[brent_data['brent_date'].dt.year >= 2024]
    
    brent_data = brent_data.sort_values('brent_date')
    print(f"Données Brent chargées, triées et filtrées à partir de 2024. Shape: {brent_data.shape}")
    return brent_data

def load_fuel_data(folder_path):
    print(f"Chargement des données de carburant depuis {folder_path}")
    all_data = []
    for filename in os.listdir(folder_path):
        if filename.endswith('.csv'):
            file_path = os.path.join(folder_path, filename)
            df = pd.read_csv(file_path)
            df['rate_date'] = pd.to_datetime(df['rate_date'])
            all_data.append(df)
    fuel_data = pd.concat(all_data, ignore_index=True)
    fuel_data = fuel_data[~fuel_data['fuel_name'].isin(['GPLc', 'E85'])]
    
    # Filtrer les données à partir de 2024
    fuel_data = fuel_data[fuel_data['rate_date'].dt.year >= 2024]
    
    print(f"Données de carburant chargées et filtrées à partir de 2024. Shape: {fuel_data.shape}")
    return fuel_data


def classify_stations(fuel_data):
    print("Classification des stations par gamme de prix")
    station_classifications = {}
    fuel_types = fuel_data['fuel_name'].unique()

    for fuel_type in fuel_types:
        fuel_type_data = fuel_data[fuel_data['fuel_name'] == fuel_type]
        station_avg_prices = fuel_type_data.groupby('id')['price'].mean().reset_index()
        
        thresholds = np.percentile(station_avg_prices['price'], [33, 66])
        
        def classify_price(price):
            if price <= thresholds[0]:
                return 'low-cost'
            elif price <= thresholds[1]:
                return 'normal'
            else:
                return 'premium'
        
        station_classifications[fuel_type] = station_avg_prices.set_index('id')['price'].apply(classify_price).to_dict()

    return station_classifications

def save_station_classifications(station_classifications, output_dir):
    classification_df = pd.DataFrame(station_classifications)
    classification_df.index.name = 'station_id'
    classification_df.reset_index(inplace=True)
    
    classification_file = os.path.join(output_dir, 'station_classifications.csv')
    classification_df.to_csv(classification_file, index=False)
    print(f"Classifications des stations sauvegardées dans {classification_file}")

class FuelPriceDataset(Dataset):
    def __init__(self, data, sequence_length, target_days):
        self.data = data
        self.sequence_length = sequence_length
        self.target_days = target_days
        print(f"Shape of data in FuelPriceDataset: {self.data.shape}")

    def __len__(self):
        return len(self.data) - self.sequence_length - max(self.target_days)

    def __getitem__(self, idx):
        x = self.data.iloc[idx:idx+self.sequence_length].values
        y = self.data.iloc[idx+self.sequence_length:idx+self.sequence_length+max(self.target_days)+1]['price'].values
        y = [y[day] for day in self.target_days]
        
        if idx == 0:  # Print only for the first item   
            print(f"Sample input (X) at index 0:")
            print(x)
            print(f"Sample output (y) at index 0:")
            print(y)
        
        return torch.FloatTensor(x), torch.FloatTensor(y)

class LSTMModel(nn.Module):
    def __init__(self, input_size, hidden_size, num_layers, output_size):
        super(LSTMModel, self).__init__()
        self.hidden_size = hidden_size
        self.num_layers = num_layers
        self.lstm = nn.LSTM(input_size, hidden_size, num_layers, batch_first=True)
        self.fc = nn.Linear(hidden_size, output_size)
        
        # Initialisation des poids
        for name, param in self.lstm.named_parameters():
            if 'weight' in name:
                init.xavier_uniform_(param)
            elif 'bias' in name:
                init.constant_(param, 0.0)
        init.xavier_uniform_(self.fc.weight)
        init.constant_(self.fc.bias, 0.0)

    def forward(self, x):
        h0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
        c0 = torch.zeros(self.num_layers, x.size(0), self.hidden_size).to(device)
        out, _ = self.lstm(x, (h0, c0))
        out = self.fc(out[:, -1, :])
        return out

def train_model(model, train_loader, val_loader, criterion, optimizer, num_epochs, patience, output_dir, fuel_type, price_range, scaler):
    train_losses = []
    val_losses = []
    best_val_loss = float('inf')
    epochs_no_improve = 0

    for epoch in range(num_epochs):
        model.train()
        train_loss = 0
        for batch_x, batch_y in train_loader:
            batch_x, batch_y = batch_x.to(device), batch_y.to(device)
            optimizer.zero_grad()
            outputs = model(batch_x)
            loss = criterion(outputs, batch_y)
            loss.backward()
            optimizer.step()
            train_loss += loss.item()
        
        model.eval()
        val_loss = 0
        with torch.no_grad():
            for batch_x, batch_y in val_loader:
                batch_x, batch_y = batch_x.to(device), batch_y.to(device)
                outputs = model(batch_x)
                loss = criterion(outputs, batch_y)
                val_loss += loss.item()
        
        train_loss /= len(train_loader)
        val_loss /= len(val_loader)
        train_losses.append(train_loss)
        val_losses.append(val_loss)
        
        print(f"Epoch [{epoch+1}/{num_epochs}], Train Loss: {train_loss:.6f}, Val Loss: {val_loss:.6f}")

        if val_loss < best_val_loss:
            best_val_loss = val_loss
            epochs_no_improve = 0
            # Sauvegarder le meilleur modèle
            torch.save(model.state_dict(), os.path.join(output_dir, f'best_model_{fuel_type}_{price_range}.pth'))
        else:
            epochs_no_improve += 1

        if epochs_no_improve == patience:
            print(f"Early stopping triggered after {epoch + 1} epochs")
            break

    # Charger le meilleur modèle avant de faire les prédictions finales
    model.load_state_dict(torch.load(os.path.join(output_dir, f'best_model_{fuel_type}_{price_range}.pth')))
    
    # Générer le graphique et calculer les métriques
    mse, mae, r2 = plot_predictions_vs_actual(model, val_loader, scaler, output_dir, fuel_type, price_range)
    
    return train_losses, val_losses, mse, mae, r2

def plot_learning_curves(train_losses, val_losses, output_dir, fuel_type, price_range):
    plt.figure(figsize=(10, 6))
    plt.plot(train_losses, label='Train Loss')
    plt.plot(val_losses, label='Validation Loss')
    plt.title(f'Learning Curves - {fuel_type} - {price_range}')
    plt.xlabel('Epochs')
    plt.ylabel('Loss')
    plt.legend()
    plt.grid(True)
    plt.tight_layout()
    plt.savefig(os.path.join(output_dir, f'learning_curves_{fuel_type}_{price_range}.png'))
    plt.close()

def plot_predictions_vs_actual(model, val_loader, scaler, output_dir, fuel_type, price_range):
    model.eval()
    predictions = []
    actual_values = []

    with torch.no_grad():
        for batch_x, batch_y in val_loader:
            batch_x = batch_x.to(device)
            outputs = model(batch_x)
            predictions.extend(outputs.cpu().numpy())
            actual_values.extend(batch_y.numpy())

    predictions = np.array(predictions)
    actual_values = np.array(actual_values)

    plt.figure(figsize=(12, 6))
    plt.scatter(actual_values[:, 0], predictions[:, 0], alpha=0.5)
    plt.plot([actual_values[:, 0].min(), actual_values[:, 0].max()], 
             [actual_values[:, 0].min(), actual_values[:, 0].max()], 
             'r--', lw=2)
    plt.xlabel('Valeurs réelles')
    plt.ylabel('Prédictions')
    plt.title(f'Prédictions vs Valeurs réelles - {fuel_type} - {price_range}')
    plt.tight_layout()
    plt.savefig(os.path.join(output_dir, f'predictions_vs_actual_{fuel_type}_{price_range}.png'))
    plt.close()

    # Calcul des métriques
    mse = np.mean((predictions[:, 0] - actual_values[:, 0])**2)
    mae = np.mean(np.abs(predictions[:, 0] - actual_values[:, 0]))
    r2 = 1 - (np.sum((actual_values[:, 0] - predictions[:, 0])**2) / 
              np.sum((actual_values[:, 0] - np.mean(actual_values[:, 0]))**2))

    print(f"MSE: {mse:.4f}")
    print(f"MAE: {mae:.4f}")
    print(f"R2 Score: {r2:.4f}")

    return mse, mae, r2


def prepare_data_for_fuel_type_and_range(merged_data, fuel_type, price_range, station_classifications, sequence_length, target_days):
    print(f"Préparation des données pour {fuel_type} - {price_range}")
    stations_in_range = [station for station, range_ in station_classifications[fuel_type].items() if range_ == price_range]
    fuel_data = merged_data[(merged_data['fuel_name'] == fuel_type) & (merged_data['id'].isin(stations_in_range))].copy()
    
    # Traitement des variables temporelles
    fuel_data['day_of_week'] = fuel_data['rate_date'].dt.dayofweek
    fuel_data['month'] = fuel_data['rate_date'].dt.month
    
    # Encodage cyclique pour le jour de la semaine et le mois
    fuel_data['day_of_week_sin'] = np.sin(2 * np.pi * fuel_data['day_of_week'] / 7)
    fuel_data['day_of_week_cos'] = np.cos(2 * np.pi * fuel_data['day_of_week'] / 7)
    fuel_data['month_sin'] = np.sin(2 * np.pi * fuel_data['month'] / 12)
    fuel_data['month_cos'] = np.cos(2 * np.pi * fuel_data['month'] / 12)
    
    # Standardisation du prix du Brent (au lieu de normaliser)
    scaler = StandardScaler()
    fuel_data['brent_rate_eur_scaled'] = scaler.fit_transform(fuel_data[['brent_rate_eur']])
    
    # Sélection des colonnes finales
    columns_to_use = ['price', 'brent_rate_eur_scaled', 'day_of_week_sin', 'day_of_week_cos', 'month_sin', 'month_cos']
    fuel_data_prepared = fuel_data[columns_to_use]
    
    print("Statistiques des données préparées:")
    print(fuel_data_prepared.describe())
    
    print("\nNombre de valeurs uniques par colonne:")
    print(fuel_data_prepared.nunique())
    
    print("\nVérification des valeurs nulles:")
    print(fuel_data_prepared.isnull().sum())

    dataset = FuelPriceDataset(fuel_data_prepared, sequence_length, target_days)
    
    train_size = int(0.8 * len(dataset))
    train_dataset, val_dataset = torch.utils.data.random_split(dataset, [train_size, len(dataset) - train_size])
    
    return train_dataset, val_dataset, scaler

def main(args):
    print("Début du processus principal")
    
    brent_data = load_brent_data(args.brent_data)
    fuel_data = load_fuel_data(args.fuel_data)

    print("Fusion des données Brent et carburant")
    merged_data = pd.merge_asof(fuel_data.sort_values('rate_date'), 
                                brent_data.reset_index().sort_values('brent_date'),
                                left_on='rate_date', 
                                right_on='brent_date',
                                direction='nearest')
    print(f"Données fusionnées. Shape: {merged_data.shape}")

    station_classifications = classify_stations(fuel_data)
    save_station_classifications(station_classifications, args.output_dir)

    price_ranges = ['low-cost', 'normal', 'premium']
    fuel_types = merged_data['fuel_name'].unique()

    for fuel_type in fuel_types:
        for price_range in price_ranges:
            print(f"\nTraitement de {fuel_type} - {price_range}")
            
            output_dir = os.path.join(args.output_dir, fuel_type, price_range)
            os.makedirs(output_dir, exist_ok=True)
            
            try:
                train_dataset, val_dataset, scaler = prepare_data_for_fuel_type_and_range(
                    merged_data, fuel_type, price_range, station_classifications, args.sequence_length, args.target_days
                )
                
                if len(train_dataset) < args.min_train_samples:
                    print(f"Pas assez de données pour {fuel_type} - {price_range}. Ignoré.")
                    continue
                
                train_loader = DataLoader(train_dataset, batch_size=args.batch_size, shuffle=True)
                val_loader = DataLoader(val_dataset, batch_size=args.batch_size, shuffle=False)
                
                print(f"Taille du dataset d'entraînement : {len(train_dataset)}")
                print(f"Taille du dataset de validation : {len(val_dataset)}")
                print(f"Nombre de batchs d'entraînement : {len(train_loader)}")
                print(f"Nombre de batchs de validation : {len(val_loader)}")
                
                sample_x, sample_y = next(iter(train_loader))
                input_size = sample_x.shape[2]
                model = LSTMModel(input_size, args.hidden_size, args.num_layers, len(args.target_days)).to(device)
                
                criterion = nn.MSELoss()
                optimizer = optim.Adam(model.parameters(), lr=args.learning_rate)

                train_losses, val_losses, mse, mae, r2 = train_model(
                    model, train_loader, val_loader, criterion, optimizer, 
                    args.num_epochs, args.patience, output_dir, fuel_type, price_range, scaler
                )

                # Sauvegarder le modèle final
                model_filename = os.path.join(output_dir, f'final_model_{fuel_type}_{price_range}.pth')
                torch.save(model.state_dict(), model_filename)

                # Sauvegarder le scaler
                scaler_filename = os.path.join(output_dir, f'scaler_{fuel_type}_{price_range}.pkl')
                pd.to_pickle(scaler, scaler_filename)

                # Sauvegarder les paramètres du modèle
                params = {
                    'input_size': input_size,
                    'hidden_size': args.hidden_size,
                    'num_layers': args.num_layers,
                    'output_size': len(args.target_days),
                    'sequence_length': args.sequence_length,
                    'target_days': args.target_days
                }
                params_filename = os.path.join(output_dir, f'model_params_{fuel_type}_{price_range}.json')
                with open(params_filename, 'w') as f:
                    json.dump(params, f)

                # Sauvegarder les métriques
                metrics = {
                    'mse': mse,
                    'mae': mae,
                    'r2': r2
                }
                metrics_filename = os.path.join(output_dir, f'metrics_{fuel_type}_{price_range}.json')
                with open(metrics_filename, 'w') as f:
                    json.dump(metrics, f)

                # Tracer et sauvegarder les courbes d'apprentissage
                plot_learning_curves(train_losses, val_losses, output_dir, fuel_type, price_range)

                print(f"Modèle, paramètres, métriques et graphiques pour {fuel_type} - {price_range} sauvegardés dans {output_dir}")

            except Exception as e:
                print(f"Erreur lors du traitement de {fuel_type} - {price_range}: {str(e)}")

    print("Processus terminé pour tous les types de carburant et gammes de prix.")

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description="Entraînement du modèle de prédiction des prix du carburant")
    parser.add_argument("--brent_data", type=str, required=True, help="Chemin vers le fichier de données Brent")
    parser.add_argument("--fuel_data", type=str, required=True, help="Chemin vers le dossier contenant les données de carburant")
    parser.add_argument("--output_dir", type=str, default="./output", help="Dossier de sortie pour les modèles et les paramètres")
    parser.add_argument("--hidden_size", type=int, default=64, help="Taille de la couche cachée LSTM")
    parser.add_argument("--num_layers", type=int, default=2, help="Nombre de couches LSTM")
    parser.add_argument("--sequence_length", type=int, default=30, help="Longueur de la séquence d'entrée")
    parser.add_argument("--target_days", nargs='+', type=int, default=[3, 7, 15, 30], help="Jours cibles pour la prédiction")
    parser.add_argument("--batch_size", type=int, default=32, help="Taille du batch pour l'entraînement")
    parser.add_argument("--num_epochs", type=int, default=50, help="Nombre d'époques d'entraînement")
    parser.add_argument("--learning_rate", type=float, default=0.001, help="Taux d'apprentissage")
    parser.add_argument("--min_train_samples", type=int, default=50, help="Nombre minimum d'échantillons d'entraînement")
    parser.add_argument("--patience", type=int, default=5, help="Nombre d'époques sans amélioration avant l'arrêt précoce")

    args = parser.parse_args()
    
    print(f"Arguments reçus: {args}")
    
    os.makedirs(args.output_dir, exist_ok=True)
    print(f"Dossier de sortie principal créé/vérifié: {args.output_dir}")

    main(args)