<commit_before>from __future__ import division
from scipy import linalg
import os
import pandas as pd
import numpy as np
# import matplotlib.pyplot as plt
# import seaborn as sns
from scipy.stats import pearsonr
from copy import deepcopy
from emotioncf.algorithms import NNMF, knn_similarity, knn_predict
from emotioncf.algorithms import nmf_multiplicative_fit, nmf_multiplicative_predict

__all__ = ['CF','create_sub_by_item_matrix','get_mse']
__author__ = ["<NAME>"]
__license__ = "MIT"

class CF(object):
    def __init__(self,
                ratings,
                n_train_items=None):

        if not isinstance(ratings,pd.DataFrame):
            raise ValueError('ratings must be a pandas dataframe instance')            
        self.ratings = ratings

        if n_train_items is None:
            self.n_train_items = None
            self.train = None
            self.test = None
        else:
            self.train_test_split(n_train_items=n_train_items)
        
        # Initialize
        self.subject_similarity = None
        self.H = None
        self.W = None
        self.nnmf_sgd = None
    
    def __repr__(self):
        return '%s.%s(rating=%s, n_train_items=%s)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.ratings.shape,
            self.n_train_items
            )

    def train_test_split(self, n_train_items=20):
        ''' Split ratings matrix into train and test items.  adds test and train matrices to CF instance.

        Args:
            train_items: number of items for test dictionary or list of specific items

        '''
        
        self.n_train_items = n_train_items
        self.train = self.ratings.copy()
        self.train.loc[:,:] = np.full(self.ratings.shape, np.nan)
        self.test = self.ratings.copy()

        for sub in self.ratings.index:
            sub_train_rating_item =  np.random.choice(self.ratings.columns,replace=False, size=n_train_items)
            self.train.loc[sub, sub_train_rating_item] = self.ratings.loc[sub, sub_train_rating_item]
            self.test.loc[sub, sub_train_rating_item] = np.nan
        assert(~np.any(self.train==self.test))

    def fit(self, method='nnmf_sgd', **kwargs):
        ''' Fit collaborative model to training data

        Args:
            method: type of algorithm to fit ['knn','nnmf_multiplicative','nnmf_sgd']
            params: dictionary of parameters to pass to algorithm

        '''
        print(method)
        print(kwargs)
        if self.train is None:
            raise ValueError('Make sure you have run train_test_split() method.')
        
        if method is 'knn':
            if 'k' not in kwargs:
                raise ValueError('Make sure you pass a valid "k".')
            self.subject_similarity = knn_similarity(self.train, **kwargs)

        if method is 'nnmf_multiplicative':
            self.W, self.H = nmf_multiplicative(self.train, n_components=params['n_factors'], max_iter=params['max_iterations'])
        
        if method is 'nnmf_sgd':
            if 'n_iterations' not in kwargs:
                raise ValueError('Make sure you pass n_iterations.')

            self.nnmf_sgd = NNMF(self.train, 
                            # mask=mask,
                            learning='sgd', 
                            **kwargs)
            self.nnmf_sgd.train(n_iterations)

    def predict(self, method='nnmf_sgd', **kwargs):
        ''' Predict new data after fitting collaborative filtering algorithm to data
        
        Args:
            method: type of algorithm to use in prediction ['knn','nnmf_multiplicative','nnmf_sgd']

        '''

        if method is 'knn':
            if self.subject_similarity is None:
                raise ValueError('Make sure you have run train method.')
            self.predicted_ratings = knn_predict(self.train, self.subject_similarity, k = params['k'])

        if method is 'nnmf_multiplicative':
            if self.H is None:
                raise ValueError('Make sure you have run train method.')

        if method is 'nnmf_sgd':
            if self.nnmf_sgd is None:
                raise ValueError('Make sure you have run train method.')
            self.predicted_ratings = self.nnmf_sgd.predict()

    def plot_predictions(self, mask=None):
        
        if self.train is None:
            raise ValueError('Need to run split_train_test() first.')

        if self.predicted_ratings is None:
            raise ValueError('Need to predict ratings first.')
        
        f, ax = plt.subplots(nrows=1,ncols=3, figsize=(15,8))
        sns.heatmap(self.ratings,vmax=100,vmin=0,ax=ax[0],square=False)
        ax[0].set_title('Actual User/Item Ratings')
        sns.heatmap(self.predicted_ratings,vmax=100,vmin=0,ax=ax[1],square=False)
        ax[1].set_title('Predicted User/Item Ratings')
        if isinstance(self.test,pd.DataFrame):
            actual = self.test.values.flatten()
        elif isinstance(self.test,np.ndarray):
            actual = self.test.flatten()
        else:
            raise ValueError('Make sure self.test are pandas data frame or numpy array.')
        if isinstance(self.predicted_ratings,pd.DataFrame):
            pred = self.predicted_ratings.values.flatten()
        elif isinstance(self.predicted_ratings,np.ndarray):
            pred = self.predicted_ratings.flatten()
        else:
            raise ValueError('Make sure predicted_ratings are pandas data frame or numpy array.')
        if mask is not None:
            if isinstance(mask,pd.DataFrame):
                mask = mask.values.flatten()
            elif isinstance(mask,np.ndarray):
                mask = mask.flatten()
            else:
                raise ValueError('Make sure mask is a pandas data frame or numpy array.')
            pred = pred[~mask]
            actual = actual[~mask]
        ax[2].scatter(actual[(~np.isnan(actual)) & (~np.isnan(pred))],pred[(~np.isnan(actual)) & (~np.isnan(pred))])
        ax[2].set_xlabel('Actual Ratings')
        ax[2].set_ylabel('Predicted Ratings')
        ax[2].set_title('Predicted Ratings')
        r = pearsonr(actual[(~np.isnan(actual)) & (~np.isnan(pred))],pred[(~np.isnan(actual)) & (~np.isnan(pred))])
        print 'Correlation: %s' % r[0]
        return f, r
    
    def subject_correlations(self, mask=None):
        '''Calculate observed/predicted correlation for each subject in matrix'''
        if mask is not None:
            r = []
            for i,sub in enumerate(self.test.index):
                r.append(pearsonr(self.test.loc[sub,mask[i,:]],self.predicted_ratings[i,mask[i,:]])[0])
        else:
            r = []
            for i,sub in enumerate(self.test.index):
                r.append(pearsonr(self.test.loc[sub,:],self.predicted_ratings[i,:])[0])
        return r

    def evaluate(pred, actual, metric='correlation'):<commit_msg>updated print statement for python3 compatibility<commit_after>from __future__ import division
from scipy import linalg
import os
import pandas as pd
import numpy as np
# import matplotlib.pyplot as plt
# import seaborn as sns
from scipy.stats import pearsonr
from copy import deepcopy
from emotioncf.algorithms import NNMF, knn_similarity, knn_predict
from emotioncf.algorithms import nmf_multiplicative_fit, nmf_multiplicative_predict

__all__ = ['CF','create_sub_by_item_matrix','get_mse']
__author__ = ["<NAME>"]
__license__ = "MIT"

class CF(object):
    def __init__(self,
                ratings,
                n_train_items=None):

        if not isinstance(ratings,pd.DataFrame):
            raise ValueError('ratings must be a pandas dataframe instance')            
        self.ratings = ratings

        if n_train_items is None:
            self.n_train_items = None
            self.train = None
            self.test = None
        else:
            self.train_test_split(n_train_items=n_train_items)
        
        # Initialize
        self.subject_similarity = None
        self.H = None
        self.W = None
        self.nnmf_sgd = None
    
    def __repr__(self):
        return '%s.%s(rating=%s, n_train_items=%s)' % (
            self.__class__.__module__,
            self.__class__.__name__,
            self.ratings.shape,
            self.n_train_items
            )

    def train_test_split(self, n_train_items=20):
        ''' Split ratings matrix into train and test items.  adds test and train matrices to CF instance.

        Args:
            train_items: number of items for test dictionary or list of specific items

        '''
        
        self.n_train_items = n_train_items
        self.train = self.ratings.copy()
        self.train.loc[:,:] = np.full(self.ratings.shape, np.nan)
        self.test = self.ratings.copy()

        for sub in self.ratings.index:
            sub_train_rating_item =  np.random.choice(self.ratings.columns,replace=False, size=n_train_items)
            self.train.loc[sub, sub_train_rating_item] = self.ratings.loc[sub, sub_train_rating_item]
            self.test.loc[sub, sub_train_rating_item] = np.nan
        assert(~np.any(self.train==self.test))

    def fit(self, method='nnmf_sgd', **kwargs):
        ''' Fit collaborative model to training data

        Args:
            method: type of algorithm to fit ['knn','nnmf_multiplicative','nnmf_sgd']
            params: dictionary of parameters to pass to algorithm

        '''
        print(method)
        print(kwargs)
        if self.train is None:
            raise ValueError('Make sure you have run train_test_split() method.')
        
        if method is 'knn':
            if 'k' not in kwargs:
                raise ValueError('Make sure you pass a valid "k".')
            self.subject_similarity = knn_similarity(self.train, **kwargs)

        if method is 'nnmf_multiplicative':
            self.W, self.H = nmf_multiplicative(self.train, n_components=params['n_factors'], max_iter=params['max_iterations'])
        
        if method is 'nnmf_sgd':
            if 'n_iterations' not in kwargs:
                raise ValueError('Make sure you pass n_iterations.')

            self.nnmf_sgd = NNMF(self.train, 
                            # mask=mask,
                            learning='sgd', 
                            **kwargs)
            self.nnmf_sgd.train(n_iterations)

    def predict(self, method='nnmf_sgd', **kwargs):
        ''' Predict new data after fitting collaborative filtering algorithm to data
        
        Args:
            method: type of algorithm to use in prediction ['knn','nnmf_multiplicative','nnmf_sgd']

        '''

        if method is 'knn':
            if self.subject_similarity is None:
                raise ValueError('Make sure you have run train method.')
            self.predicted_ratings = knn_predict(self.train, self.subject_similarity, k = params['k'])

        if method is 'nnmf_multiplicative':
            if self.H is None:
                raise ValueError('Make sure you have run train method.')

        if method is 'nnmf_sgd':
            if self.nnmf_sgd is None:
                raise ValueError('Make sure you have run train method.')
            self.predicted_ratings = self.nnmf_sgd.predict()

    def plot_predictions(self, mask=None):
        
        if self.train is None:
            raise ValueError('Need to run split_train_test() first.')

        if self.predicted_ratings is None:
            raise ValueError('Need to predict ratings first.')
        
        f, ax = plt.subplots(nrows=1,ncols=3, figsize=(15,8))
        sns.heatmap(self.ratings,vmax=100,vmin=0,ax=ax[0],square=False)
        ax[0].set_title('Actual User/Item Ratings')
        sns.heatmap(self.predicted_ratings,vmax=100,vmin=0,ax=ax[1],square=False)
        ax[1].set_title('Predicted User/Item Ratings')
        if isinstance(self.test,pd.DataFrame):
            actual = self.test.values.flatten()
        elif isinstance(self.test,np.ndarray):
            actual = self.test.flatten()
        else:
            raise ValueError('Make sure self.test are pandas data frame or numpy array.')
        if isinstance(self.predicted_ratings,pd.DataFrame):
            pred = self.predicted_ratings.values.flatten()
        elif isinstance(self.predicted_ratings,np.ndarray):
            pred = self.predicted_ratings.flatten()
        else:
            raise ValueError('Make sure predicted_ratings are pandas data frame or numpy array.')
        if mask is not None:
            if isinstance(mask,pd.DataFrame):
                mask = mask.values.flatten()
            elif isinstance(mask,np.ndarray):
                mask = mask.flatten()
            else:
                raise ValueError('Make sure mask is a pandas data frame or numpy array.')
            pred = pred[~mask]
            actual = actual[~mask]
        ax[2].scatter(actual[(~np.isnan(actual)) & (~np.isnan(pred))],pred[(~np.isnan(actual)) & (~np.isnan(pred))])
        ax[2].set_xlabel('Actual Ratings')
        ax[2].set_ylabel('Predicted Ratings')
        ax[2].set_title('Predicted Ratings')
        r = pearsonr(actual[(~np.isnan(actual)) & (~np.isnan(pred))],pred[(~np.isnan(actual)) & (~np.isnan(pred))])
        print('Correlation: %s' % r[0])
        return f, r
    
    def subject_correlations(self, mask=None):
        '''Calculate observed/predicted correlation for each subject in matrix'''
        if mask is not None:
            r = []
            for i,sub in enumerate(self.test.index):
                r.append(pearsonr(self.test.loc[sub,mask[i,:]],self.predicted_ratings[i,mask[i,:]])[0])
        else:
            r = []
            for i,sub in enumerate(self.test.index):
                r.append(pearsonr(self.test.loc[sub,:],self.predicted_ratings[i,:])[0])
        return r

    def evaluate(pred, actual, metric='correlation'):
{'fraction_non_alphanumeric': 0.10640608034744843, 'fraction_numerical': 0.003836409699601882, 'mean_word_length': 2.247003525264395, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6492632', 'n_tokens_mistral': 4187, 'n_tokens_neox': 4088, 'n_words': 973}