<jupyter_start><jupyter_text>Functions<jupyter_code>%cd /content/fancyimpute/test/
from low_rank_data import XY, XY_incomplete, missing_mask
from common import reconstruction_error
XY_incomplete
%cd /content/fancyimpute/fancyimpute/
!apt-get install python3-nose
import pandas as pd
import os
import numpy as np
import warnings
import logging
import importlib
from sklearn.utils import check_array
from nose.tools import eq_

from fancyimpute.knn import KNN


from keras import regularizers
from keras.callbacks import EarlyStopping
from keras.layers import Input
from keras.models import Model
from sklearn.utils import shuffle, check_array

from scaler import Scaler
from keras_models import KerasMatrixFactorizer


#import required packages
from sklearn import neighbors
from sklearn.model_selection import KFold
from sklearn.metrics import mean_squared_error
from sklearn.model_selection import GridSearchCV
from math import sqrt
import matplotlib.pyplot as plt
%matplotlib inline
def import_from(module, name):
    '''
    usage example:
    grid = import_from('sklearn.model_selection', 'GridSearchCV')
    is equivalent to:
    from sklearn.model_selection import GridSearchV as grid
    '''
    module = importlib.import_module(module)
    return getattr(module, name)
def generate_random_column_samples(column):
    col_mask = np.isnan(column)
    n_missing = np.sum(col_mask)
    if n_missing == len(column):
        logging.warn("No observed values in column")
        return np.zeros_like(column)

    mean = np.nanmean(column)
    std = np.nanstd(column)

    if np.isclose(std, 0):
        return np.array([mean] * n_missing)
    else:
        return np.random.randn(n_missing) * std + mean

class Solver(object):
    def __init__(
            self,
            fill_method="zero",
            min_value=None,
            max_value=None,
            normalizer=None):
        self.fill_method = fill_method
        self.min_value = min_value
        self.max_value = max_value
        self.normalizer = normalizer

    def __repr__(self):
        return str(self)

    def __str__(self):
        field_list = []
        for (k, v) in sorted(self.__dict__.items()):
            if v is None or isinstance(v, (float, int)):
                field_list.append("%s=%s" % (k, v))
            elif isinstance(v, str):
                field_list.append("%s='%s'" % (k, v))
        return "%s(%s)" % (
            self.__class__.__name__,
            ", ".join(field_list))

    def _check_input(self, X):
        if len(X.shape) != 2:
            raise ValueError("Expected 2d matrix, got %s array" % (X.shape,))

    def _check_missing_value_mask(self, missing):
        if not missing.any():
            warnings.simplefilter("always")
            warnings.warn("Input matrix is not missing any values")
        if missing.all():
            raise ValueError("Input matrix must have some non-missing values")

    def _fill_columns_with_fn(self, X, missing_mask, col_fn):
        for col_idx in range(X.shape[1]):
            missing_col = missing_mask[:, col_idx]
            n_missing = missing_col.sum()
            if n_missing == 0:
                continue
            col_data = X[:, col_idx]
            fill_values = col_fn(col_data)
            if np.all(np.isnan(fill_values)):
                fill_values = 0
            X[missing_col, col_idx] = fill_values

    def fill(
            self,
            X,
            missing_mask,
            fill_method=None,
            inplace=False):
        """
        Parameters
        ----------
        X : np.array
            Data array containing NaN entries

        missing_mask : np.array
            Boolean array indicating where NaN entries are

        fill_method : str
            "zero": fill missing entries with zeros
            "mean": fill with column means
            "median" : fill with column medians
            "min": fill with min value per column
            "random": fill with gaussian samples according to mean/std of column

        inplace : bool
            Modify matrix or fill a copy
        """
        X = check_array(X, force_all_finite=False)

        if not inplace:
            X = X.copy()

        if not fill_method:
            fill_method = self.fill_method

        if fill_method not in ("zero", "mean", "median", "min", "random"):
            raise ValueError("Invalid fill method: '%s'" % (fill_method))
        elif fill_method == "zero":
            # replace NaN's with 0
            X[missing_mask] = 0
        elif fill_method == "mean":
            self._fill_columns_with_fn(X, missing_mask, np.nanmean)
        elif fill_method == "median":
            self._fill_columns_with_fn(X, missing_mask, np.nanmedian)
        elif fill_method == "min":
            self._fill_columns_with_fn(X, missing_mask, np.nanmin)
        elif fill_method == "random":
            self._fill_columns_with_fn(
                X,
                missing_mask,
                col_fn=generate_random_column_samples)
        return X

    def prepare_input_data(self, X):
        """
        Check to make sure that the input matrix and its mask of missing
        values are valid. Returns X and missing mask.
        """
        X = check_array(X, force_all_finite=False)
        if X.dtype != "f" and X.dtype != "d":
            X = X.astype(float)

        self._check_input(X)
        missing_mask = np.isnan(X)
        self._check_missing_value_mask(missing_mask)
        return X, missing_mask

    def clip(self, X):
        """
        Clip values to fall within any global or column-wise min/max constraints
        """
        X = np.asarray(X)
        if self.min_value is not None:
            X[X < self.min_value] = self.min_value
        if self.max_value is not None:
            X[X > self.max_value] = self.max_value
        return X

    def project_result(self, X):
        """
        First undo normalization and then clip to the user-specified min/max
        range.
        """
        X = np.asarray(X)
        if self.normalizer is not None:
            X = self.normalizer.inverse_transform(X)
        return self.clip(X)

    def solve(self, X, missing_mask):
        """
        Given an initialized matrix X and a mask of where its missing values
        had been, return a completion of X.
        """
        raise ValueError("%s.solve not yet implemented!" % (
            self.__class__.__name__,))

    def fit_transform(self, X, y=None):
        """
        Fit the imputer and then transform input `X`

        Note: all imputations should have a `fit_transform` method,
        but only some (like IterativeImputer in sklearn) also support inductive
        mode using `fit` or `fit_transform` on `X_train` and then `transform`
        on new `X_test`.
        """
        X_original, missing_mask = self.prepare_input_data(X)
        observed_mask = ~missing_mask
        X = X_original.copy()
        if self.normalizer is not None:
            X = self.normalizer.fit_transform(X)
        X_filled = self.fill(X, missing_mask, inplace=True)
        if not isinstance(X_filled, np.ndarray):
            raise TypeError(
                "Expected %s.fill() to return NumPy array but got %s" % (
                    self.__class__.__name__,
                    type(X_filled)))

        X_result = self.solve(X_filled, missing_mask)
        if not isinstance(X_result, np.ndarray):
            raise TypeError(
                "Expected %s.solve() to return NumPy array but got %s" % (
                    self.__class__.__name__,
                    type(X_result)))

        X_result = self.project_result(X=X_result)
        X_result[observed_mask] = X_original[observed_mask]
        return X_result

    def fit(self, X, y=None):
        """
        Fit the imputer on input `X`.

        Note: all imputations should have a `fit_transform` method,
        but only some (like IterativeImputer in sklearn) also support inductive
        mode using `fit` or `fit_transform` on `X_train` and then `transform`
        on new `X_test`.
        """
        raise ValueError(
            "%s.fit not implemented! This imputation algorithm likely "
            "doesn't support inductive mode. Only fit_transform is "
            "supported at this time." % (
                self.__class__.__name__,))

    def transform(self, X, y=None):
        """
        Transform input `X`.

        Note: all imputations should have a `fit_transform` method,
        but only some (like IterativeImputer in sklearn) also support inductive
        mode using `fit` or `fit_transform` on `X_train` and then `transform`
        on new `X_test`.
        """
        raise ValueError(
            "%s.transform not implemented! This imputation algorithm likely "
            "doesn't support inductive mode. Only %s.fit_transform is "
            "supported at this time." % (
                self.__class__.__name__, self.__class__.__name__))

class MatrixFactorization(Solver):
    """
    Given an incomplete (m,n) matrix X, factorize it into
    U, V where U.shape = (m, k) and V.shape = (k, n).

    The U, V are found by minimizing the difference between U.dot.V and
    X at the observed entries along with a sparsity penalty for U and an
    L2 penalty for V.
    """

    def __init__(
            self,
            rank=10,
            learning_rate=0.001,
            epochs=10000,
            patience=5,
            l2_penalty=1e-5,
            use_bias=True,
            min_improvement=0.001,
            optimization_algorithm="Adam",
            loss='mse',
            validation_frac=0.1,
            min_value=None,
            max_value=None,
            normalizer=Scaler(),
            verbose=True):
        Solver.__init__(
            self,
            min_value=min_value,
            max_value=max_value,
            normalizer=normalizer)
        self.rank = rank
        self.learning_rate = learning_rate
        self.epochs = epochs
        self.patience = patience
        self.l2_penalty = l2_penalty
        self.use_bias = use_bias
        self.optimization_algorithm = optimization_algorithm
        self.loss = loss
        self.validation_frac = validation_frac
        self.min_improvement = min_improvement
        self.normalizer = normalizer
        self.verbose = verbose

    def solve(self, X, missing_mask):
        X = check_array(X, force_all_finite=False)

        # shape data to fit into keras model
        (n_samples, n_features) = X.shape
        observed_mask = ~missing_mask
        missing_mask_flat = missing_mask.flatten()
        observed_mask_flat = observed_mask.flatten()

        columns, rows = np.meshgrid(np.arange(n_features), np.arange(n_samples))

        # training data
        i_tr = rows.flatten()[observed_mask_flat]
        j_tr = columns.flatten()[observed_mask_flat]
        ij_tr = np.vstack([i_tr, j_tr]).T  # input to factorizer
        y_tr = X.flatten()[observed_mask_flat]  # output of factorizer
        ij_tr, y_tr = shuffle(ij_tr, y_tr)

        # make a keras model
        main_input = Input(shape=(2,), dtype='int32')
        embed = KerasMatrixFactorizer(
            rank=self.rank,
            input_dim_i=n_samples,
            input_dim_j=n_features,
            embeddings_regularizer=regularizers.l2(self.l2_penalty),
            use_bias=self.use_bias
        )(main_input)
        model = Model(inputs=main_input, outputs=embed)
        optimizer = import_from(
            'keras.optimizers', self.optimization_algorithm
        )(lr=self.learning_rate)
        model.compile(optimizer=optimizer, loss=self.loss)
        callbacks = [EarlyStopping(patience=self.patience, min_delta=self.min_improvement)]
        model.fit(
            ij_tr,
            y_tr,
            batch_size=int(len(y_tr) * (1 - self.validation_frac)),
            epochs=self.epochs,
            validation_split=self.validation_frac,
            callbacks=callbacks,
            shuffle=True,
            verbose=self.verbose
        )

        # reassemble the original X
        i_ts = rows.flatten()[missing_mask_flat]
        j_ts = columns.flatten()[missing_mask_flat]
        ij_ts = np.vstack([i_ts, j_ts]).T  # input to factorizer
        X[i_ts, j_ts] = model.predict(ij_ts).T[0]

        return X<jupyter_output><empty_output><jupyter_text>Performance Model<jupyter_code>def test_matrix_factorization_with_low_rank_random_matrix():
    solver = MatrixFactorization(
        learning_rate=0.01,
        rank=3,
        l2_penalty=0,
        min_improvement=1e-6)
    #print(solver)
    XY_completed = solver.fit_transform(XY_incomplete)
    #print(XY_completed)
    _, missing_mae = reconstruction_error(
        XY,
        XY_completed,
        missing_mask,
        name="MatrixFactorization")
    assert missing_mae < 0.1, "Error too high!"
    return XY_completed, solver 

X_filled = test_matrix_factorization_with_low_rank_random_matrix()<jupyter_output>[1;30;43mSe han truncado las últimas 5000 líneas del flujo de salida.[0m
Epoch 2612/10000
1/1 [==============================] - 0s 79ms/step - loss: 4.5226e-05 - val_loss: 0.0558
Epoch 2613/10000
1/1 [==============================] - 0s 79ms/step - loss: 4.5263e-05 - val_loss: 0.0557
Epoch 2614/10000
1/1 [==============================] - 0s 82ms/step - loss: 4.5319e-05 - val_loss: 0.0558
Epoch 2615/10000
1/1 [==============================] - 0s 91ms/step - loss: 4.5323e-05 - val_loss: 0.0557
Epoch 2616/10000
1/1 [==============================] - 0s 78ms/step - loss: 4.5345e-05 - val_loss: 0.0558
Epoch 2617/10000
1/1 [==============================] - 0s 80ms/step - loss: 4.5316e-05 - val_loss: 0.0557
Epoch 2618/10000
1/1 [==============================] - 0s 166ms/step - loss: 4.5306e-05 - val_loss: 0.0558
Epoch 2619/10000
1/1 [==============================] - 0s 77ms/step - loss: 4.5243e-05 - val_loss: 0.0557
Epoch 2620/10000
1/1 [==============================] - 0s 81ms/step[...]<jupyter_text>Iterative Super Vector Machine<jupyter_code>from fancyimpute import IterativeSVD

def test_iterative_svd_with_low_rank_random_matrix():
    solver = IterativeSVD(rank=3)
    XY_completed = solver.fit_transform(XY_incomplete)
    _, missing_mae = reconstruction_error(
        XY,
        XY_completed,
        missing_mask,
        name="IterativeSVD")
    assert missing_mae < 0.1, "Error too high!"


X_filled_svd = test_iterative_svd_with_low_rank_random_matrix()
def test_knn():
    # get a baseline error from just zero-filling the missing entries
    sad_zero_fill = np.sum(np.abs(XY[missing_mask]))
    mad_zero_fill = sad_zero_fill / missing_mask.sum()
    print("MAD zero-fill = ", mad_zero_fill)
    for k in [5, 15, 30]:
        print("-- k=", k)
        XY_completed = KNN(k).fit_transform(XY_incomplete)
        mask = np.isfinite(XY_completed)
        eq_((~mask).sum(), 0)
        diff = (XY_completed - XY)[missing_mask]
        sad = np.sum(np.abs(diff))
        mad = sad / missing_mask.sum()
        print("Mean absolute error", mad)
        # knnImpute should be at least twice as good as just zero fill
        assert mad <= (mad_zero_fill / 2.0), \
            "Expected knnImpute to be 2x better than zeroFill (%f) but got MAD=%f" % (
                mad_zero_fill,
                mad)
X_filled_knn = test_knn()<jupyter_output>MAD zero-fill =  1.1593901478619075
-- k= 5
Imputing row 1/500 with 4 missing, elapsed time: 0.063
Imputing row 101/500 with 4 missing, elapsed time: 0.070
Imputing row 201/500 with 4 missing, elapsed time: 0.078
Imputing row 301/500 with 3 missing, elapsed time: 0.085
Imputing row 401/500 with 4 missing, elapsed time: 0.093
Mean absolute error 0.5122055775761624
-- k= 15
Imputing row 1/500 with 4 missing, elapsed time: 0.053
Imputing row 101/500 with 4 missing, elapsed time: 0.059
Imputing row 201/500 with 4 missing, elapsed time: 0.065
Imputing row 301/500 with 3 missing, elapsed time: 0.069
Imputing row 401/500 with 4 missing, elapsed time: 0.077
Mean absolute error 0.4832376639047212
-- k= 30
Imputing row 1/500 with 4 missing, elapsed time: 0.058
Imputing row 101/500 with 4 missing, elapsed time: 0.065
Imputing row 201/500 with 4 missing, elapsed time: 0.073
Imputing row 301/500 with 3 missing, elapsed time: 0.081
Imputing row 401/500 with 4 missing, elapsed time: 0.089
Mean absolu[...]<jupyter_text>Handling NAN values in quality air sensors<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
# path = '/content/drive/MyDrive/IMT Atlantique/Project_S5_Air/Data_csv/DataIlias/'
# pathdir  = np.array(os.listdir(path))
# csv_file_list = {}


# for dirname in pathdir:
#   csv_file_list[dirname] = pd.read_csv(path + dirname, header=0, index_col=0).reset_index()


# super_df = pd.concat(csv_file_list.values(), keys=csv_file_list.keys())
# super_df.reset_index(inplace=True)
# super_df.rename(columns={'level_0': 'Sensor'}, inplace=True)
# super_df
# super_df.drop(columns=['level_1'], inplace=True)
# super_df[super_df[['PM2_MIN', 'PM2_MAX', 'PM2_MOY']] == 400] = np.nan
# super_df.to_csv('/content/drive/MyDrive/IMT Atlantique/Project_S5_Air/Data_csv/superdfi.csv')
super_df = pd.read_csv('/content/drive/MyDrive/IMT Atlantique/Project_S5_Air/air-polution-sensor/data/superdfi.csv', header=0, index_col=0)
filldf = super_df.copy(deep=True)
filldf.drop(columns=['Sensor','Date', 'Hour'], inplace=True)
X_incomplete = filldf.to_numpy()
solverSensors = MatrixFactorization(
        learning_rate=0.01,
        rank=3,
        l2_penalty=0,
        min_improvement=1e-6)
X_completed = solverSensors.fit_transform(X_incomplete)
print(X_incomplete)
print(X_completed)
super_dfFilled = pd.DataFrame(X_completed, columns = ['PM2_MIN','PM2_MAX','PM2_MOY'])
super_df.head()
super_dfFilled[['Sensor', 'Date', 'Hour']] = super_df[['Sensor', 'Date', 'Hour']]
super_dfFilled = super_dfFilled[['Sensor', 'Date', 'Hour', 'PM2_MIN', 'PM2_MAX', 'PM2_MOY']]
super_dfFilled
super_dfFilled.isnull().sum().sum(  )<jupyter_output><empty_output><jupyter_text>Performance in interval of time series without NAN values<jupyter_code>super_df = pd.read_csv('/content/drive/MyDrive/IMT Atlantique/Project_S5_Air/air-polution-sensor/data/superdfi.csv', header=0, index_col=0)
#aux = super_df[(super_df['Sensor'] == 'sensor14v2.csv')]
aux = super_df[(super_df['Sensor'] == 'sensor14v2.csv') | (super_df['Sensor'] == 'sensor13v2.csv') | (super_df['Sensor'] == 'sensor12v2.csv') | (super_df['Sensor'] == 'sensor11v2.csv') | (super_df['Sensor'] == 'sensor10v2.csv') | (super_df['Sensor'] == 'sensor9v2.csv') | (super_df['Sensor'] == 'sensor8v2.csv') | (super_df['Sensor'] == 'sensor7v2.csv') ]
#aux =aux.dropna().reset_index()
#aux.isnull().sum().sum()

aux = pd.pivot_table(data=aux,values='PM2_MOY',index=['Date', 'Hour'],columns= 'Sensor')
aux = aux.dropna()
columnst = aux.columns
indext = aux.index
aux_arr = aux.to_numpy()
aux_incomplete =  np.copy(aux_arr)
missing_raw_values = np.random.uniform(0, 1, aux_arr.shape)
missing_mask = missing_raw_values < 0.3
missing_mask[0] = False
aux_incomplete[missing_mask] = np.nan

aux_incomplete = pd.DataFrame(aux_incomplete, columns = columnst, index= indext).reset_index()
auxoriginal = aux_incomplete.copy(deep=True)

aux_incomplete
 df = aux_incomplete.iloc[:, 2:10]
 

df= df.fillna(df.interpolate(method='linear'), axis=1)
#aux_incomplete = aux_incomplete.dropna()
arr_completed = df.to_numpy()
df.isnull().sum()
rmse, missing_mae = reconstruction_error(
        aux_arr,
        arr_completed,
        missing_mask,
        name="SoftImpute")
plt.figure(figsize=(20,5))
plt.axis([0,1000, 0, 23])
plt.plot(df['sensor11v2.csv'],'b')
plt.title('Average PM2.5 filled by Linear Interpolation , when the percentaje of missing values equals 25%, sensor 11', size=20)
plt.xlabel('Time',size=18)
plt.ylabel('Average PM2.5',size=18)
plt.figure(figsize=(20,5))
plt.axis([0,1000, 0, 23])
plt.plot(auxoriginal['sensor11v2.csv'],'r')
plt.title('Original average PM2.5 with a percentaje of missing values equals 25%, sensor 11 ', size=20)
plt.xlabel('Time',size=18)
plt.ylabel('Average PM2.5',size=18)
def test_matrix_completion():
    solverTestSensors = MatrixFactorization(
        learning_rate=0.001,
        rank=2,
        l2_penalty= 1e-5,
        patience=100,
        validation_frac=0.15,
        min_improvement=1e-6)
    aux_completed = solverTestSensors.fit_transform(aux_incomplete)
    #print(XY_completed)
    _, missing_mae = reconstruction_error(
        aux_arr,
        aux_completed,   
        missing_mask,
        name="MatrixFactorization")
    #assert missing_mae < 0.1, "Error too high!"
    return aux_completed 
aux_completed = test_matrix_completion()
from fancyimpute import IterativeSVD

def test_iterative_svd_with_low_rank_random_matrix():
    solver = IterativeSVD(rank=2)
    aux_completed = solver.fit_transform(aux_incomplete)
    _, missing_mae = reconstruction_error(
        aux_arr,
        aux_completed,
        missing_mask,
        name="IterativeSVD")
    #assert missing_mae < 1.2, "Error too high!"
    return aux_completed


aux_completedSVD = test_iterative_svd_with_low_rank_random_matrix()
from fancyimpute import SoftImpute

def test_soft_impute_with_low_rank_random_matrix():
    solver = SoftImpute()
    aux_completed = solver.fit_transform(aux_incomplete)
    _, missing_mae = reconstruction_error(
        aux_arr,
        aux_completed,
        missing_mask,
        name="SoftImpute")
    #assert missing_mae < 0.1, "Error too high!"
    return aux_completed

aux_completed3= test_soft_impute_with_low_rank_random_matrix()
aux_completedSI= pd.DataFrame(aux_completed3, columns = columnsp, index= indixp)
aux_completedSI =aux_completed33.reset_index()
aux_completedSI['sensor10v2.csv'].plot()


aux_completed= pd.DataFrame(aux_completed, columns = columnst, index= indext)
aux_completed =aux_completed.reset_index()

plt.figure(figsize=(20,5))
plt.axis([0,1500, 0, 70])
plt.plot(aux_completed['sensor10v2.csv'],'b')
plt.title('Average PM2 filled by Matrix Completion', size=15)
plt.xlabel('time',size=15)
plt.ylabel('PM2 Moyenne',size=15)

aux = aux.reset_index()

plt.figure(figsize=(20,5))
plt.axis([0,1500, 0, 70])
plt.plot(aux['sensor10v2.csv'],'r')
plt.title('Average PM2 filled by Matrix Completion', size=15)
plt.xlabel('time',size=15)
plt.ylabel('PM2 Moyenne',size=15)

aux_completed2 = pd.DataFrame(aux_completed2, columns = ['PM2_MAX','PM2_MOY'])
aux_completed2[['Sensor', 'Date', 'Hour', 'PM2_MIN']] = aux[['Sensor', 'Date', 'Hour', 'PM2_MIN']]
aux_completed2 = aux_completed2[['Sensor', 'Date', 'Hour', 'PM2_MIN', 'PM2_MAX', 'PM2_MOY']]
aux_completed2['Date'] = pd.to_datetime(aux_completed2['Date'])
aux['Date'] = pd.to_datetime(aux['Date'])
aux_completed2 = aux_completed2.sort_values(by='Date')
aux = aux.sort_values(by='Date')
aux_completed3 = pd.DataFrame(aux_completed3, columns = ['PM2_MAX','PM2_MOY'])
aux_completed3[['Sensor', 'Date', 'Hour', 'PM2_MIN']] = aux[['Sensor', 'Date', 'Hour', 'PM2_MIN']]
aux_completed3 = aux_completed3[['Sensor', 'Date', 'Hour', 'PM2_MIN', 'PM2_MAX', 'PM2_MOY']]
aux_completed3['Date'] = pd.to_datetime(aux_completed3['Date'])
aux['Date'] = pd.to_datetime(aux['Date'])
aux_completed3 = aux_completed3.sort_values(by='Date')
aux = aux.sort_values(by='Date')
d =pd.DataFrame(aux_incomplete, columns=['PM2_Max', 'PM2_MOY'])
d['PM2_MOY'].isnull().sum().sum()
aux['PM2_MOY'].shape
print(aux_completed2.describe())
np.isnan(aux_incomplete).sum()

plt.figure(figsize=(20,5))
plt.axis([0,5200, 0, 70])
plt.plot(arrcompletedknn['PM2_MOY'],'bo')
plt.title('Average PM2 filled by Matrix Completion', size=15)
plt.xlabel('time',size=15)
plt.ylabel('PM2 Moyenne',size=15)



plt.figure(figsize=(20,5))
plt.axis([0,5200, 0, 70])
plt.plot(aux['PM2_MOY'], 'ro')
plt.title('Average PM2 Original', size=15)
plt.xlabel('time', size=15)
plt.ylabel('PM2 Moyenne', size=15)
aux['PM2_MOY'].max()
aux_completed33['Date'] = pd.to_datetime(aux_completed33['Date'])
aux['Date'] = pd.to_datetime(aux['Date'])
aux_completed33 = aux_completed33.sort_values(by='Date')
aux = aux.sort_values(by='Date')
# Affichage
def plot_df(df, x, y, title="qu'est-ce ?", xlabel='Date', ylabel='Value', dpi=100):
    plt.figure(figsize=(16,5), dpi=dpi)
    plt.plot(x, y)
    plt.gca().set(title=title, xlabel=xlabel, ylabel=ylabel)
    plt.show()
plot_df(aux_completed33, x=aux_completed33['Date'], y=aux_completed33['sensor10v2.csv'],title='PM2MAX par heure  - RTE 01/11/2018 - 28/10/2019')

plot_df(aux, x=aux['Date'], y=aux['PM2_MOY'],title='PM2MAX par heure  - RTE 01/11/2018 - 28/10/2019')
from fancyimpute import (NuclearNormMinimization)
def test_nuclear_norm_minimization_with_low_rank_random_matrix():
    solver = NuclearNormMinimization(max_iters=2000)
    XY_completed = solver.fit_transform(aux_incomplete[0:4000])
    _, missing_mae = reconstruction_error(
        aux_arr[0:4000], XY_completed, missing_mask[0:4000], name="NuclearNorm")
    #assert missing_mae < 0.1, "Error too high!"
    return XY_completed
 aux_completed = test_nuclear_norm_minimization_with_low_rank_random_matrix()
aux_completed = pd.DataFrame(aux_completed, columns = ['PM2_MIN','PM2_MAX','PM2_MOY'])
aux_completed[['Sensor', 'Date', 'Hour']] = aux[['Sensor', 'Date', 'Hour']]
aux_completed = aux_completed[['Sensor', 'Date', 'Hour', 'PM2_MIN', 'PM2_MAX', 'PM2_MOY']]
aux_completed['Date'] = pd.to_datetime(aux_completed['Date'])
aux['Date'] = pd.to_datetime(aux['Date'])
super_df['PM2_MOY'].describe()
super_df['PM2_MOY'].isnull().sum().sum()
(8510 * 100 ) / 55474<jupyter_output><empty_output><jupyter_text>Nuevo codigo MF<jupyter_code>%matplotlib inline
import matplotlib
import matplotlib.pyplot as plt
import numpy as np
import pandas as pd
from scipy.interpolate import Rbf, InterpolatedUnivariateSpline

def reconstruction_error(XY, XY_completed, missing_mask, name=None):
    """
    Returns mean squared error and mean absolute error for
    completed matrices.
    """
    value_pairs = [
        (i, j, XY[i, j], XY_completed[i, j])
        for i in range(XY.shape[0])
        for j in range(XY.shape[1])
        if missing_mask[i, j]
    ]
    print("First 10 reconstructed values:")
    for (i, j, x, xr) in value_pairs[:10]:
        print("  (%d,%d)  %0.4f ~= %0.4f" % (i, j, x, xr))
    diffs = [actual - predicted for (_, _, actual, predicted) in value_pairs]
    missing_mse = np.sqrt(np.mean([diff ** 2 for diff in diffs]))
    missing_mae = np.mean([np.abs(diff) for diff in diffs])
    print("%s RMSE: %0.4f, MAE: %0.4f" % (
        "" if not name else name + " ",
        missing_mse,
        missing_mae))
    return missing_mse, missing_mae
super_df = pd.read_csv('/content/drive/MyDrive/IMT Atlantique/Project_S5_Air/air-polution-sensor/data/superdfi.csv', header=0, index_col=0)
aux = super_df[(super_df['Sensor'] == 'sensor14v2.csv') | (super_df['Sensor'] == 'sensor13v2.csv') | (super_df['Sensor'] == 'sensor12v2.csv') | (super_df['Sensor'] == 'sensor11v2.csv') | (super_df['Sensor'] == 'sensor10v2.csv') | (super_df['Sensor'] == 'sensor9v2.csv') | (super_df['Sensor'] == 'sensor8v2.csv') | (super_df['Sensor'] == 'sensor7v2.csv') ]

aux = pd.pivot_table(data=aux,values='PM2_MOY',index=['Date', 'Hour'],columns='Sensor' )
aux = aux.fillna(aux.mean())
columnst = aux.columns
indext = aux.index

aux_arr = aux.to_numpy()
aux_incomplete =  np.copy(aux_arr)
missing_raw_values = np.random.uniform(0, 1, aux_arr.shape)
missing_mask = missing_raw_values < 0.2
aux_incomplete[missing_mask] = np.nan
aux_incomplete = pd.DataFrame(aux_incomplete, columns = columnst, index= indext).reset_index()

aux_incomplete.iloc[:, 2:10] = aux_incomplete.iloc[:, 2:10].interpolate(method='polynomial', order=5)
arr_completed = aux_incomplete.iloc[:, 2:10].to_numpy()
rmse, missing_mae = reconstruction_error(
        aux.to_numpy(),
        arr_completed,
        missing_mask,
        name="SoftImpute")
a = aux['sensor8v2.csv']
plt.figure(figsize=(20,5))
plt.axis([0,1500, 0, 110])
plt.plot(a['sensor8v2.csv'].reset_index(),'b')
plt.title('Average PM2 filled by Matrix Completion', size=15)
plt.xlabel('time',size=15)
plt.ylabel('PM2 Moyenne',size=15)
b = aux_incomplete['sensor8v2.csv'].reset_index()
plt.figure(figsize=(20,5))
plt.axis([0,1500, 0, 110])
plt.plot(b['sensor8v2.csv'],'b')
plt.title('Average PM2 filled by Matrix Completion', size=15)
plt.xlabel('time',size=15)
plt.ylabel('PM2 Moyenne',size=15)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13032538260030332, 'fraction_numerical': 0.03395146835792086, 'mean_word_length': 3.04249686498537, 'pattern_counts': {'":': 10, '<': 33, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '291204', 'license': 'MIT', 'n_tokens_mistral': 10653, 'n_tokens_neox': 9643, 'n_words': 2086, 'path': 'examples/fancyimpute.ipynb', 'repo_name': 'daca1897/Air-polution-Sensor', 'size': '29012'}