<jupyter_start><jupyter_text>Arquivo de Refactoring do Modelo<jupyter_code>#from pandas import DataFrame, Series
import pandas as pd

import numpy as np

import time

from sklearn.model_selection import train_test_split
from sklearn.metrics import confusion_matrix, accuracy_score
# Importa o código principal
import sys
sys.path.append('C:\\Users\\<NAME>\\Desktop\\Qualificação PPGCC\\abordagem\\RFNS')

# from grimoire.EnginneringForest import EnginneringForest
from grimoire.ClassifierEnginneringForest import ClassifierEnginneringForest
from grimoire.LoggerEnginnering import LoggerEnginnering
df_heart = pd.read_csv('../datasets/heart.csv',
                       engine='c', 
                       memory_map=True, 
                       low_memory=True)

X=df_heart[['age', 'sex', 'cp', 'trestbps',  'chol', 'fbs', 'restecg', 
            'thalach', 'exang', 'oldpeak', 'slope', 'ca', 'thal']]
# Labels
y=df_heart['target']

X_train, X_test, y_train, y_test = train_test_split(X, y, test_size=0.30, 
                                                        random_state=100, 
                                                        shuffle=True, 
                                                        stratify=y)
X_train.shape
print(list(X_train.index))<jupyter_output>[241, 246, 97, 60, 298, 217, 42, 88, 7, 143, 126, 52, 187, 152, 64, 68, 194, 164, 282, 247, 18, 165, 111, 239, 156, 193, 148, 46, 45, 142, 99, 202, 40, 183, 184, 210, 284, 188, 205, 27, 288, 297, 200, 124, 37, 106, 78, 41, 132, 31, 80, 243, 190, 294, 245, 206, 220, 154, 289, 162, 105, 237, 275, 110, 213, 19, 95, 3, 287, 59, 4, 84, 71, 238, 234, 171, 159, 32, 293, 253, 161, 235, 75, 145, 69, 54, 240, 266, 141, 15, 216, 79, 29, 236, 219, 185, 35, 221, 296, 177, 85, 197, 109, 12, 6, 153, 103, 291, 8, 233, 98, 255, 121, 56, 91, 283, 286, 76, 226, 186, 62, 222, 58, 175, 107, 10, 176, 150, 207, 118, 279, 120, 250, 229, 163, 271, 264, 270, 50, 211, 260, 280, 26, 16, 23, 157, 276, 230, 299, 261, 160, 182, 100, 292, 149, 96, 198, 20, 131, 204, 65, 208, 290, 102, 70, 195, 262, 209, 43, 215, 138, 125, 47, 257, 191, 137, 123, 104, 39, 170, 242, 254, 61, 203, 168, 155, 134, 228, 227, 267, 17, 13, 86, 89, 249, 281, 21, 128, 139, 136, 36, 302, 112, 49, 5, 1, 72, 201, 151, 269, 9, 83]<jupyter_text>Procura os erros de indices nas etapas de treinamento<jupyter_code>class EnginneringForest(ClassifierEnginneringForest):
    
    __slots__ = ('estimators_', 'select_features_', 'group_features_', 
                 'df_predict_', 'n_features_', 'n_samples_', 'name_features_',
                 'prefix_column_predict', 'logger', 'blockpredict', 'autoclean')
    
    def __init__(self, select_features: int, reset_log=False, name_log='enginnering.log'):
        if type(select_features) != int:
            raise TypeError('Expectd value int in select_features')
        
        self.estimators_ = []
        self.select_features_ = select_features
        self.group_features_ = []
        self.df_predict_ = []
        self.n_features_ = 0
        self.n_samples_ = 0
        self.name_features_ = []
        self.logger = LoggerEnginnering(name='enginnering', 
                                        log_file=name_log,
                                        drop_old=reset_log)
        self.blockpredict = []
        self.autoclean = False
        super().__init__()
        
    def __del__(self):
        del self.estimators_
        del self.select_features_
        del self.group_features_
        del self.df_predict_
        del self.n_features_
        del self.n_samples_
        del self.name_features_
        #del self.prefix_column_predict

    def build(self, features_set: list) -> None:
        """ Cria um vetor com o número de árvores igual ao número de 
            subconjuntos possíveis """
        self.group_features_ = self.get_arrangement_features(features_set,
                                                             self.select_features_)
        self.group_features_ = self.get_pack_nparray(self.group_features_)
        n_estimator = len(self.group_features_)
        self.estimators_ = self.make_lote_base_estimator(n_estimator)
        self.estimators_ = self.get_pack_nparray(self.estimators_)
        
        #print('N group features: ', len(self.group_features_))
        #print('N estimators: ', len(self.estimators_))
        #print('Group features: \n{0}'.format(self.group_features_))

    def train(self, group_feature: list, estimator):
        msg = 'Training subset = {0}, Timing = {1}, Size (Kb) = {2}'
        start_train = time.time()
        
        subset_xdata, subset_ydata = self.get_subset(group_feature)
        fit_ = estimator.fit(subset_xdata, subset_ydata)
        
        #print('Train = {0}, Length X = {1}, Length Y = {2}'.format(group_feature, subset_xdata.shape, subset_ydata.shape))
        
        #del subset_xdata
        #del subset_ydata
        
        end_train = time.time()
        self.logger.add('debug',msg.format(group_feature, 
                                           (end_train - start_train),
                                           self.get_size_estimator(fit_)))
        
        return fit_

    def fit(self, X, y) -> None:
        from pandas import DataFrame, Series
        if not isinstance(X, DataFrame):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
        if not isinstance(y, Series):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
            
        self.n_samples_, self.n_features_ = X.shape
        self.name_features_ = X.columns
        
        self.train_X = X
        self.train_y = y
        
        #print('N samples: ',  self.n_samples_)
        #print('N features: ', self.n_features_)
        #print('Name features: ', self.name_features_)
        
        # Cria a floresta
        self.build(features_set=self.name_features_)

        # Treina as arvores individualmente
        self.estimators_ = [self.train(subset_feature, estimator) 
                            for subset_feature, 
                                estimator in zip(self.group_features_, 
                                                 self.estimators_)]
        self.estimators_ = self.get_pack_nparray(self.estimators_)
        
        if self.autoclean:
            del self.train_X
            del self.train_y
model_eg = EnginneringForest(3)
model_eg.fit(X_train, y_train)
subset_x, subset_y = model_eg.get_subset(['sex','restecg','thalach'])
subset_x.head()
subset_y.head()
print(subset_x.shape, subset_y.shape)
for x, y in zip(subset_x.index, subset_y.index):
                print('{0} == {1}'.format(x,y))<jupyter_output>241 == 241
246 == 246
97 == 97
60 == 60
298 == 298
217 == 217
42 == 42
88 == 88
7 == 7
143 == 143
126 == 126
52 == 52
187 == 187
152 == 152
64 == 64
68 == 68
194 == 194
164 == 164
282 == 282
247 == 247
18 == 18
165 == 165
111 == 111
239 == 239
156 == 156
193 == 193
148 == 148
46 == 46
45 == 45
142 == 142
99 == 99
202 == 202
40 == 40
183 == 183
184 == 184
210 == 210
284 == 284
188 == 188
205 == 205
27 == 27
288 == 288
297 == 297
200 == 200
124 == 124
37 == 37
106 == 106
78 == 78
41 == 41
132 == 132
31 == 31
80 == 80
243 == 243
190 == 190
294 == 294
245 == 245
206 == 206
220 == 220
154 == 154
289 == 289
162 == 162
105 == 105
237 == 237
275 == 275
110 == 110
213 == 213
19 == 19
95 == 95
3 == 3
287 == 287
59 == 59
4 == 4
84 == 84
71 == 71
238 == 238
234 == 234
171 == 171
159 == 159
32 == 32
293 == 293
253 == 253
161 == 161
235 == 235
75 == 75
145 == 145
69 == 69
54 == 54
240 == 240
266 == 266
141 == 141
15 == 15
216 == 216
79 == 79
29 == 29
236 == 236
219 == 219
185 == 185
35 == 35
221 == [...]<jupyter_text>Procura os erros de indices na etapa de predição<jupyter_code>class EnginneringForest(ClassifierEnginneringForest):
    
    __slots__ = ('estimators_', 'select_features_', 'group_features_', 
                 'df_predict_', 'n_features_', 'n_samples_', 'name_features_',
                 'prefix_column_predict', 'logger', 
                 'blockpredict', 'autoclean')
    
    def __init__(self, select_features: int, reset_log=False, name_log='enginnering.log'):
        if type(select_features) != int:
            raise TypeError('Expectd value int in select_features')
        
        self.estimators_ = []
        self.select_features_ = select_features
        self.group_features_ = []
        self.df_predict_ = []
        self.n_features_ = 0
        self.n_samples_ = 0
        self.name_features_ = []
        self.logger = LoggerEnginnering(name='enginnering', 
                                        log_file=name_log,
                                        drop_old=reset_log)
        self.blockpredict = []
        self.autoclean = False
        super().__init__()
        
    def __del__(self):
        del self.estimators_
        del self.select_features_
        del self.group_features_
        del self.df_predict_
        del self.n_features_
        del self.n_samples_
        del self.name_features_

    def build(self, features_set: list) -> None:
        """ Cria um vetor com o número de árvores igual ao número de 
            subconjuntos possíveis """
        self.group_features_ = self.get_arrangement_features(features_set,
                                                             self.select_features_)
        self.group_features_ = self.get_pack_nparray(self.group_features_)
        n_estimator = len(self.group_features_)
        self.estimators_ = self.make_lote_base_estimator(n_estimator)
        self.estimators_ = self.get_pack_nparray(self.estimators_)
        
    def train(self, group_feature: list, estimator):
        msg = 'Training subset = {0}, Timing = {1}, Size (Kb) = {2}'
        start_train = time.time()
        
        subset_xdata, subset_ydata = self.get_subset(group_feature)
        fit_ = estimator.fit(subset_xdata, subset_ydata)
        
        end_train = time.time()
        self.logger.add('debug',msg.format(group_feature, 
                                           (end_train - start_train),
                                           self.get_size_estimator(fit_)))
        
        return fit_

    def fit(self, X, y) -> None:
        from pandas import DataFrame, Series
        if not isinstance(X, DataFrame):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
        if not isinstance(y, Series):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
            
        self.n_samples_, self.n_features_ = X.shape
        self.name_features_ = X.columns
        self.train_X = X
        self.train_y = y
        
        self.build(features_set=self.name_features_)

        self.estimators_ = [self.train(subset_feature, estimator) 
                            for subset_feature, 
                                estimator in zip(self.group_features_, 
                                                 self.estimators_)]
        self.estimators_ = self.get_pack_nparray(self.estimators_)
        
        if self.autoclean:
            del self.train_X
            del self.train_y
    
    def predict(self, X) -> list:
        from pandas import DataFrame
        if not isinstance(X, DataFrame):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
       
        self.predict_X = X

        self.logger.add('debug',
                        'Size predict = {0}, N estimators = {1}'.format(self.predict_X.shape, 
                                                                        len(self.estimators_)))                                  
        
        for x_, y_ in self.get_block_fit():
            self.logger.add('debug','Block Limit = ({}, {})'.format(x_, y_))
            
            dfsub = self.predict_X.iloc[x_:y_+1]
            block_predict = []
            
            print('Size DfSub = {0}'.format(dfsub.shape))
            
            for subset_feature, estimator in zip(self.group_features_, self.estimators_):
                self.logger.add('debug', 'Subset predict = {0}'.format(subset_feature))
                subset_test = dfsub.loc[:, subset_feature]
                block_predict.append(estimator.predict(subset_test))
            
            block_predict = np.matrix(block_predict)
            self.logger.add('debug', "Shape One = {0}".format(block_predict.shape))
            
            block_predict = block_predict.T
            self.logger.add('debug', "Shape Two = {0}".format(block_predict.shape))
            self.logger.add('debug', "Block predict \n{0}".format(block_predict))
            
            # Guarda a matrix de predição
            #np.savetxt('dump_sf_{0}_bp_{1}'.format(self.select_features_, (x_,y_)), 
            #           block_predict, delimiter=",")
            
            self.blockpredict = pd.DataFrame(block_predict)
            
            # chama o voting na matriz de predições
            block_voting = self.voting(block_predict)
            self.logger.add('debug', "Block voting data \n{0}".format(str(block_voting)))
            self.logger.add('debug', "Block voting len {0}".format(len(block_voting)))
            self.df_predict_.extend(block_voting)
        
        return self.df_predict_
    
    def voting(self, data) -> list:
        final_predict = []
        for instance in data:
            marjotiry = (instance.sum() >= (instance.shape[1] / 2)) and 1 or 0
            final_predict.append(marjotiry)
        return final_predict
model = EnginneringForest(3, name_log='test2.txt')
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
model.blockpredict.head()
print(list(y_pred))
print(list(y_test))
print(list(model.blockpredict.iloc[0]))
len(y_pred)
list(model.blockpredict.iloc[0]).count(1)
list(model.blockpredict.iloc[0]).count(0)
(model.blockpredict.iloc[0].sum() >= (model.blockpredict.iloc[0].shape[1] / 2)) and 1 or 0
mac = accuracy_score(y_test, y_pred)
print("Accuracy:", mac)
model.blockpredict.iloc[6].shape
model_eg = EnginneringForest(3)
model_eg.fit(X_train, y_train)
y_pred = model_eg.predict(X_test)
model_eg.blockpredict.head()
model_eg.blockpredict.shape
block = model_eg.blockpredict
block.insert(loc=286, column='predict', value=y_pred)
block.insert(loc=287, column='real', value=y_test)
block.head()
block = X_test
block.head()
block.shape
len(y_test)
type(y_test)
print(list(y_pred.index))
block.head()
print(list(X_test.index))
print(list(y_test.index))
len(X_test.index)
X_test.iloc[10:20]<jupyter_output><empty_output><jupyter_text>Procura os erros no voting<jupyter_code>class EnginneringForest(ClassifierEnginneringForest):
    
    __slots__ = ('estimators_', 'select_features_', 'group_features_', 
                 'df_predict_', 'n_features_', 'n_samples_', 'name_features_',
                 'prefix_column_predict', 'logger', 
                 'blockpredict', 'autoclean', 'matrix_predict')
    
    def __init__(self, select_features: int, reset_log=False, name_log='enginnering.log'):
        if type(select_features) != int:
            raise TypeError('Expectd value int in select_features')
        
        self.estimators_ = []
        self.select_features_ = select_features
        self.group_features_ = []
        self.df_predict_ = []
        self.n_features_ = 0
        self.n_samples_ = 0
        self.name_features_ = []
        self.logger = LoggerEnginnering(name='enginnering', 
                                        log_file=name_log,
                                        drop_old=reset_log)
        self.blockpredict = []
        self.matrix_predict = [[]]
        self.autoclean = False
        super().__init__()
        
    def __del__(self):
        del self.estimators_
        del self.select_features_
        del self.group_features_
        del self.df_predict_
        del self.n_features_
        del self.n_samples_
        del self.name_features_

    def build(self, features_set: list) -> None:
        """ Cria um vetor com o número de árvores igual ao número de 
            subconjuntos possíveis """
        self.group_features_ = self.get_arrangement_features(features_set,
                                                             self.select_features_)
        self.group_features_ = self.get_pack_nparray(self.group_features_)
        n_estimator = len(self.group_features_)
        self.estimators_ = self.make_lote_base_estimator(n_estimator)
        self.estimators_ = self.get_pack_nparray(self.estimators_)
        
    def train(self, group_feature: list, estimator):
        msg = 'Training subset = {0}, Timing = {1}, Size (Kb) = {2}'
        start_train = time.time()
        
        subset_xdata, subset_ydata = self.get_subset(group_feature)
        fit_ = estimator.fit(subset_xdata, subset_ydata)
        
        end_train = time.time()
        self.logger.add('debug',msg.format(group_feature, 
                                           (end_train - start_train),
                                           self.get_size_estimator(fit_)))
        
        return fit_

    def fit(self, X, y) -> None:
        from pandas import DataFrame, Series
        if not isinstance(X, DataFrame):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
        if not isinstance(y, Series):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
            
        self.n_samples_, self.n_features_ = X.shape
        self.name_features_ = X.columns
        self.train_X = X
        self.train_y = y
        
        self.build(features_set=self.name_features_)

        self.estimators_ = [self.train(subset_feature, estimator) 
                            for subset_feature, 
                                estimator in zip(self.group_features_, 
                                                 self.estimators_)]
        self.estimators_ = self.get_pack_nparray(self.estimators_)
        
        if self.autoclean:
            del self.train_X
            del self.train_y
    
    def predict(self, X) -> list:
        from pandas import DataFrame
        if not isinstance(X, DataFrame):
            raise TypeError('Expected value should descend from pandas.core.frame.DataFrame')
       
        self.predict_X = X

        self.logger.add('debug',
                        'Size predict = {0}, N estimators = {1}'.format(self.predict_X.shape, 
                                                                        len(self.estimators_)))                                  
        
        for x_, y_ in self.get_block_fit():
            self.logger.add('debug','Block Limit = ({}, {})'.format(x_, y_))
            
            dfsub = self.predict_X.iloc[x_:y_+1]
            block_predict = []
            
            print('Size DfSub = {0}'.format(dfsub.shape))
            
            for subset_feature, estimator in zip(self.group_features_, self.estimators_):
                self.logger.add('debug', 'Subset predict = {0}'.format(subset_feature))
                subset_test = dfsub.loc[:, subset_feature]
                block_predict.append(estimator.predict(subset_test))
            
            block_predict = np.matrix(block_predict)
            self.matrix_predict = block_predict
            self.logger.add('debug', "Shape One = {0}".format(block_predict.shape))
            
            block_predict = block_predict.T
            self.logger.add('debug', "Shape Two = {0}".format(block_predict.shape))
            self.logger.add('debug', "Block predict \n{0}".format(block_predict))
            
            # Guarda a matrix de predição
            #np.savetxt('dump_sf_{0}_bp_{1}'.format(self.select_features_, (x_,y_)), 
            #           block_predict, delimiter=",")
            
            self.blockpredict = pd.DataFrame(block_predict)
            
            # chama o voting na matriz de predições
            block_voting = self.voting(block_predict)
            self.logger.add('debug', "Block voting data \n{0}".format(str(block_voting)))
            self.logger.add('debug', "Block voting len {0}".format(len(block_voting)))
            self.df_predict_.extend(block_voting)
        
        return self.df_predict_
    
    def voting(self, data) -> list:
        final_predict = []
        i = 0
        for instance in data:
            cz = instance.tolist()[0].count(0)
            co = instance.tolist()[0].count(1)
            marjotiry = (co > cz) and 1 or 0
            final_predict.append(marjotiry)
            counter = instance.tolist()[0].count(1)
            print('Instance = {0}, Marjority = {1}, Type = {2}, Shape = {3}, Sum = {4}, N one = {5}'.format(i, marjotiry, type(instance), instance.shape, instance.sum(), counter))
            i = i + 1
        return final_predict
model = EnginneringForest(3, name_log='test2.txt')
model.fit(X_train, y_train)
y_pred = model.predict(X_test)
model.matrix_predict[0].tolist()[0].count(1)
model.matrix_predict[0]
for instance in model.matrix_predict:
    print("\n", instance.tolist()[0])<jupyter_output>[0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 0, 1]

 [0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 0, 0, 1, 0, 1, 0, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 0, 1, 1, 1, 1, 0, 1, 1, 0, 1, 0, 1, 1, 0, 0, 1, 0, 0, 1, 0, 1, 0, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1]

 [0, 1, 1, 0, 1, 1, 1, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 1, 1, 0, 0, 1, 0, 0, 1, 1, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 1, 1, 0, 0, 1, 0, 1, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 1, 0, 1, 0, 1, 1, 0, 1, 0, 1]

 [0, 1, 1, 0, 1, 0, 0, 1, 0, 0, 1, 1, 0, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 0, 0, 1, 0, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 0, 1, 0, 1, 1, 1[...]
{'fraction_non_alphanumeric': 0.12324007220216607, 'fraction_numerical': 0.06787003610108304, 'mean_word_length': 2.080483736447039, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 46, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '412393', 'license': 'Apache-2.0', 'n_tokens_mistral': 8511, 'n_tokens_neox': 7079, 'n_words': 1796, 'path': 'jupyter/Refactoring Grimoiri Spell Cod 01.ipynb', 'repo_name': 'darosantos/RFNS', 'size': '22167'}