<jupyter_start><jupyter_text>Experiment F2) Implementar um framework de busca de hiperparâmetros.2.1) Parâmetros específicos para cada método de processamento do ds.2.2) N top colunas (`top_cols`) do dataset.2.3) Parâmetro $L$ (`recomender(...,L,...)`).5) Implementar como entrada uma empresa nova, conter mapeamento de valores.Talvez criar o notebook da Second_View, com:1) Verificar `sklearn.inspection.permutation_importance`.- Author: <NAME> [\[e-mail\]](mailto:'Israel%20Oliveira%20')<jupyter_code>%load_ext watermark
from typing import NewType, List
from loguru import logger
import pandas as pd
import numpy as np
from time import time
from scipy.spatial.distance import cosine
from sklearn.decomposition import FactorAnalysis, FastICA, PCA, IncrementalPCA, NMF, TruncatedSVD
from collections import defaultdict, Counter
import functools
import operator
from copy import deepcopy
from tqdm import tqdm
# Run this cell before close.
%watermark -d --iversion -b -r -g -m -v
logger.info("Carregando dataset...")

path_data = '../data/'
df_marked = pd.read_csv(path_data+'estaticos_market.csv')
df_marked = df_marked.drop(columns=['Unnamed: 0'])
logger.info("...pronto!")

logger.info("Carregando dataset de validação...")

df_ep_list = [pd.read_csv(path_data+'estaticos_portfolio{}.csv'.format(i+1)) for i in range(3)]
tmp = []
for i in range(3):
    df_ep_list[i]['P'] = i+1 
    tmp.append(df_ep_list[i][['id','P']])
df_ep = pd.concat(tmp)
del df_ep_list
del tmp
logger.info("...pronto!")

# Para desenvolvimento do framework:
#df_marked = df_marked.merge(df_ep, on='id').drop(columns=['P'])
def flat(a):
    return functools.reduce(operator.iconcat, a, []) 

def feat_proc(dataset = df_marked, col_target = 'id', feat_cols = df_marked.columns[1:], N_topcols = -1):
    logger.info("Processando as features...")
    missing_count = {}
    remove_cols = []
    for col in feat_cols:
        try:
            missing_count[col] = sum(dataset[col].isna()) / dataset[col].nunique()
            dataset[col] = dataset[col].fillna(0)*1
        except ZeroDivisionError:
            remove_cols.append(col)

    feat_cols = [col for col in feat_cols if col not in remove_cols]

    def normalize(x):
        return (x-np.min(x))/(np.max(x) - np.min(x)) if (np.max(x) - np.min(x)) > 0 else (x-np.min(x))

    for col in feat_cols:
        try:
            dataset[col] = normalize(dataset[col].tolist())
        except:
            maping = {val:i+1 for i,val in enumerate(dataset[col].unique())}
            dataset[col] = dataset[col].apply(lambda x: maping[x])
            dataset[col] = normalize(dataset[col].tolist())

    remove_cols = []
    for col in feat_cols:
        if dataset[col].nunique() == 1:
            remove_cols.append(col)
    feat_cols = [col for col in feat_cols if col not in remove_cols]
    N_topcols = N_topcols if (N_topcols > 0) and (N_topcols <= len(feat_cols)) else -1
    feat_cols_vals = [(col,val) for col,val in list(missing_count.items()) if col in feat_cols]
    if N_topcols == -1:
        top_cols = feat_cols
    else:
        top_cols = [col for col,_ in sorted(feat_cols_vals, key=lambda x: x[1])[:N_topcols]]
    
    missing_count = {key:val for key,val in missing_count.items() if col in feat_cols }
    logger.info("...pronto!")
    return dataset[[col_target]+top_cols], missing_count



def Manhattan(X,vec):
    return abs(X - vec).sum(-1)

def Camberra(X,vec):
    return abs((X - vec)/(X + vec)).sum(-1)

def BrayCurtis(X,vec):
    return abs((X - vec)).sum(-1) / abs((X - vec)).sum(-1).sum(-1)

def np_cossine(X,vec):
    return np.array([sum(X[i]*vec) / sum(X[i]**2)*sum(vec**2) for i in range(X.shape[0])])

def npj_cossine(X,vec):
    return npj.array([sum(X[i]*vec) / sum(X[i]**2)*sum(vec**2) for i in range(X.shape[0])])

def scy_cossine(X,vec):
    return np.array([cosine(X[i],vec) for i in range(X.shape[0])])

#Manhattanj = jit(Manhattan)
#Camberraj = jit(Camberra)
#BrayCurtisj = jit(BrayCurtis)
#np_cossinej = jit(npj_cossine)

dist_func = [Manhattan, Camberra, BrayCurtis, np_cossine, scy_cossine]
#tmp = [Manhattanj, Camberraj, BrayCurtisj, np_cossinej]

#for dist in tmp:
#    dist.__name__ += 'j'
#
#dist_func = dist_func+tmp
#del tmp

def Nothing(arg):
    return arg

def npSVD(M):
    u, _, _ = np.linalg.svd(M, full_matrices=False)
    return u

#def npSVDj(M):
#    u, _, _ = npj.linalg.svd(M, full_matrices=False)
#    return u

# Mais rápido!
#_npSVDj = jit(npSVDj)


def _PCA(M,n_components=None):
    out = PCA(n_components=n_components)
    return out.fit_transform(M)

def _FastICA(M,n_components=None):
    out = FastICA(n_components=n_components)
    return out.fit_transform(M)

def _FactorAnalysis(M,n_components=None):
    out = FactorAnalysis(n_components=n_components)
    return out.fit_transform(M)

def _IncrementalPCA(M,n_components=None):
    out = IncrementalPCA(n_components=n_components)
    return out.fit_transform(M)

def _TruncatedSVD(M,n_components=None):
    out = TruncatedSVD(n_components=n_components)
    return out.fit_transform(M)

def _NMF(M,n_components=None):
    out = NMF(n_components=n_components)
    return out.fit_transform(M)

redux_func = [Nothing, npSVD, _NMF, _TruncatedSVD, _IncrementalPCA, _FactorAnalysis, _FastICA, _PCA]

data, normalize = feat_proc()

def escalaropt_missings(df: pd.DataFrame, score: dict):
    df_score = pd.DataFrame(score.items(), columns=['col','score'])
    df_score = pd.DataFrame(missing_count.items(), columns=['col','score'])
    df_score['escala_opt'] = 1-normalize((np.sqrt(df_score.score)))
    #df_score['escala_opt'].sort_values().reset_index(drop=True).plot()
    #df_score['escala_opt'].apply(lambda x: max(x,0.1)).sort_values().reset_index(drop=True).plot()
    df_score['escala_opt'] = df_score['escala_opt'].apply(lambda x: max(x,0.1))
    for row in df_score.iterrows():
        df[row.col] = row.escala_opt*df[row.col]
    return df

def escalaropt_std(df: pd.DataFrame, score: dict):
    df_score = pd.DataFrame(score.items(), columns=['col','score'])
    df_score = pd.DataFrame(missing_count.items(), columns=['col','score'])
    df_score['escala_opt'] = normalize([np.sqrt(np.sqrt(np.sqrt(df[col].std()))) for col in df_score['col']])
    #df_score['escala_opt'].sort_values().reset_index(drop=True).plot()
    #df_score['escala_opt'].apply(lambda x: max(x,0.1)).sort_values().reset_index(drop=True).plot()
    df_score['escala_opt'] = df_score['escala_opt'].apply(lambda x: max(x,0.1))
    for row in df_score.iterrows():
        df[row.col] = row.escala_opt*df[row.col]
    return df

def escalaropt_entropy(df: pd.DataFrame, score: dict):
    df_score = pd.DataFrame(score.items(), columns=['col','score'])
    df_score = pd.DataFrame(missing_count.items(), columns=['col','score'])
    df_score['escala_opt'] = normalize([(-sum((df[col]+1)*np.log(df[col]+1))) for col in df_score['col']])
    #df_score['escala_opt'].sort_values().reset_index(drop=True).plot()
    #df_score['escala_opt'].apply(lambda x: max(x,0.1)).sort_values().reset_index(drop=True).plot()
    df_score['escala_opt'] = df_score['escala_opt'].apply(lambda x: max(x,0.1))
    for row in df_score.iterrows():
        df[row.col] = row.escala_opt*df[row.col]
    return df

procDS_func = [Nothing, escalaropt_missings, escalaropt_std, escalaropt_entropy]

Uid = NewType('uid', int)
Raw = NewType('raw', str)

class ExMatrix():
    """
        ************
    """
    def __init__(self,process_values = Nothing, factorize = Nothing, vector_distance = Manhattan, stateless: bool = False):
        self.matrix_dict = {}
        self.stateless = stateless
        self.M = None
        self.pu = None
        self.raw = None
        self.uid = None
        self.vector_distance = vector_distance
        self.factorize = factorize
        self.process_values = Nothing

    def fit(self,dataset: pd.DataFrame):
        """
            ...
        """
        self.raw = dataset[dataset.columns[0]].to_dict()
        self.uid = {raw:uid for uid,raw in self.raw.items()}
        self.all_raw = dataset[dataset.columns[0]].tolist()
        self.all_uid = dataset.index
        dataset = self.process_values(dataset)
        ds_size = len(dataset[dataset.columns[1:]].values)
        self.M = self.factorize(dataset[dataset.columns[1:]].values)
        if ds_size != self.M.shape[0]:
            raise ValueError('A fatoração não está correta!')
        del dataset
        
    def _get_neighbors(self,uid: Uid, k: int = 1, black_list: List[Uid] = []) -> List[Uid]:
        """
            Calcula todas as distâncias entre 'uid' de entrada e todos os outros 'uid'.
            A distância calciulada é armazenda e não calculada novamente. 
        """
        k = k if k >= 0 else 0
        #logger.info("Calculando todos os vizinhos...")
        #for uid2 in tqdm(self.trainset.all_users()):
        if uid not in self.matrix_dict.keys():
            self.matrix_dict[uid] = self.vector_distance(self.M,self.M[uid])
        out = [x[0] for x in sorted(
            [
                (uid2, self.matrix_dict[uid][uid2])
                for uid2 in self.all_uid
                if (uid2 not in black_list)
            ], key=lambda x: x[1])][:k]
        if self.stateless:
            del self.matrix_dict
            self.matrix_dict = {}
        return out
    
    def _uid2raw(self, uid: Uid)-> str:
        '''
            uid -> raw.
            Valor interno para externo, o nome original do usuário.
        '''
        return self.raw[uid]
    
    def _raw2uid(self, raw: Raw)-> int:
        '''
            raw -> uid.
            Valor externo para interno, o id interno do usuários..
        '''
        return self.uid[raw]
    
    def recomender(self, in_list: List[Raw], k: int = 1, L: int = 3, Fk: int = 1, limit: int = 10)-> List[Raw]:
        '''
            Faz as recomendacoes.
            ##### Função incompleta #####
        '''
        # Pega quantas recomendações por usuário em `in_list`,
        # mas sem deixar faltar
        N_in = len(in_list)
        k = k if k > 0 else 1
        R_per_in = L*(k//N_in + min(k%N_in,1))

        # Pega os `uid`
        uid_in_list = [self._raw2uid(raw) for raw in in_list]

        # Pega os vizinhos mais próximos de cada uid de entrada.
        done = False
        flag = True
        Rounds = 0
        while limit and (not done):
            Rounds += 1
            # Ele sempre pega todos novamente.
            recomendations_list = [self._get_neighbors(uid,R_per_in,uid_in_list) for uid in uid_in_list]
            # Quando limit = 0, encerra.
            limit -= 1
            # Quando tem gente o suficiente, encerra.
            if len(set(flat(recomendations_list))) >= Fk*k:
                done = True
            # Depois do primeiro loop, pega um a mais.
            R_per_in += 1

        # Aqui gera um dicionário ordenando por votacao.
        count_rec = Counter(flat(recomendations_list)) # A votação!!
        count_rec = list(count_rec.items())
        ct_pos = defaultdict(list)
        #ct_pos_inv = defaultdict(list)
        while count_rec:
            tmp = count_rec.pop(0)
            ct_pos[tmp[1]].append(tmp[0])
            #ct_pos_inv[tmp[0]].append(tmp[1])

        # Aqui considera a posiçao de vizinhos mais proximos.
        #nn_pos = defaultdict(list)
        nn_pos_inv = defaultdict(list)
        tmp = deepcopy(recomendations_list)
        while tmp:
            tmp2 = tmp.pop(0)
            n = 0
            while tmp2:
                n += 1
                tmp3 = tmp2.pop(0)
                #nn_pos[n].append(tmp3)
                nn_pos_inv[tmp3].append(n)

        # Vai separando por votação e ordem de proximidade como desempate.      
        votos_list = list(ct_pos.keys())
        out_uid = []
        while votos_list and k:
            votos = max(votos_list)
            votos_list.remove(votos)
            tmp = sorted([(tmp, min(nn_pos_inv[tmp])) for tmp in ct_pos[votos]], key=lambda x: x[1])
            while tmp and k:
                out_uid.append(tmp.pop(0)[0])
                k -= 1

        # converte para Raw e "joga fora".
        return [self._uid2raw(uid) for uid in out_uid]

def Search(N=1, process_values = Nothing, factorize = Nothing, vector_distance_list = [Manhattan]):
    ex_algo = ExMatrix(process_values = process_values, factorize = factorize)
    ex_algo.fit(data)

    out = {}
    for dist in vector_distance_list:
        ex_algo.vector_distance = dist
        print(dist.__name__)
        tmp ={1: [0], 2: [0], 3: [0]}
        t = time()
        for row in tqdm(df_ep.iterrows()):
            recs = ex_algo.recomender([row[1].id],k=N)
            tmp[row[1].P].append(any([x in df_ep.loc[df_ep.P == row[1].P].id.to_list() for x in recs])*1)
        t = time()-t
        out[dist.__name__] = {i: (sum(val)/max(1,len(val)), sum(val), len(val)) for i,val in tmp.items()}
        out[dist.__name__]['t'] = t
    return out
dist_list = [BrayCurtis] #[Manhattan, scy_cossine, Camberra, BrayCurtis, np_cossine] #[Manhattan, Camberra, BrayCurtis] #dist_func
proc_list = procDS_func #procDS_func #[Nothing] #procDS_func
redux_list = [_NMF] #redux_func #[Nothing] #[Nothing, npSVD, _NMF, _PCA, _FactorAnalysis] #redux_func
n_components_dict = {Nothing.__name__ : False,
                  #_npSVDj.__name__: False,
                  npSVD.__name__: False,
                  _NMF.__name__ : True,
                  _TruncatedSVD.__name__ : True,
                  _IncrementalPCA.__name__ : True,
                  _FactorAnalysis.__name__ : True,
                  _FastICA.__name__ : True,
                  _PCA.__name__ : True}
results_csv_name = 'Results_RC1.csv'
df_laoded = pd.read_csv(results_csv_name)
def save_results(results,df_e):
    df = pd.DataFrame(results, columns=['pre_proc','redux_func','n_components','dist','t','P1pp','P1_True','P1_len','P2pp','P2_True','P2_len','P3pp','P3_True','P3_len'])
    out = pd.concat([df_e, df])
    #df_laoded.df_laodedto_csv('Results_redux_prepro.csv',index=False)
    return out
n_components_list = [n for n in range(30,80,2)]+[n for n in range(80,150,5)]
n=0
for proc in proc_list:
    n += 1
    print(n)
    for redux in redux_list:
        if n_components_dict[redux.__name__]:
            for n_components in n_components_list:
                cond = (df_laoded['pre_proc'] == proc.__name__) & (df_laoded['redux_func'] == redux.__name__) & (df_laoded['n_components'] == n_components)
                print("Done?: {}.".format([proc.__name__, redux.__name__, n_components]))
                if sum(cond) == 0:
                    def redux_tmp(M):
                        return redux(M,n_components=n_components)
                    tmp = Search(process_values = proc, factorize = redux_tmp, vector_distance_list = dist_list)
                    results = [[proc.__name__, redux.__name__, n_components] + r for r in [[key]+[tmp[key]['t']]+flat([list(tmp[key][i+1]) for i in range(3)]) for key in tmp.keys()]]
                    df_laoded = save_results(results,df_laoded)
                    df_laoded.to_csv(results_csv_name,index=False)
                else:
                    print("Done: {}.".format([proc.__name__, redux.__name__, n_components]))
        else:
            n_components = data.shape[1]
            cond = (df_laoded['pre_proc'] == proc.__name__) & (df_laoded['redux_func'] == redux.__name__) & (df_laoded['n_components'] == n_components)
            print("Done?: {}.".format([proc.__name__, redux.__name__, n_components]))
            if sum(cond) == 0:
                tmp = Search(process_values = proc, factorize = redux, vector_distance_list= dist_list)
                results = [[proc.__name__, redux.__name__, n_components] + r for r in [[key]+[tmp[key]['t']]+flat([list(tmp[key][i+1]) for i in range(3)]) for key in tmp.keys()]]
                df_laoded = save_results(results,df_laoded)
                df_laoded.to_csv(results_csv_name,index=False)
            else:
                print("Done: {}.".format([proc.__name__, redux.__name__, n_components]))
#df = pd.DataFrame(columns=['pre_proc','redux_func','n_components','dist','t','P1pp','P1_True','P1_len','P2pp','P2_True','P2_len','P3pp','P3_True','P3_len'])
#df.to_csv(results_csv_name,index=False)
df_laoded.sort_values(by='t')
np.array([3, 4])
ex_algo = ExMatrix()
ex_algo.fit(data)
type(ex_algo.M[0])
np.ndarray
data.to_xarray()
BrayCurtis(ex_algo.M[:10],ex_algo.M[0]).shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14421263913387264, 'fraction_numerical': 0.00997506234413965, 'mean_word_length': 2.9495556089358637, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '362162', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 5953, 'n_tokens_neox': 5830, 'n_words': 1199, 'path': 'projeto/EDA/Experiment_F.ipynb', 'repo_name': 'ysraell/aceleradev_private', 'size': '16450'}