<jupyter_start><jupyter_text>EDA<jupyter_code>df.head()
df['# label'].value_counts()
df['# label'].value_counts(normalize=True)<jupyter_output><empty_output><jupyter_text>Scaling and SkewNeed to scale feature 'mass'.  Other features are already scaled or pretty close.Several features are highly skewed with min or max values > 4 standard devations from the mean<jupyter_code>feature_summary = df.iloc[:, 1:29].describe().T
feature_summary<jupyter_output><empty_output><jupyter_text>Higly skewed features<jupyter_code>right_skew = feature_summary.loc[feature_summary['max'] > feature_summary['mean'] + feature_summary['std']*4]
left_skew = feature_summary.loc[feature_summary['min'] < feature_summary['mean'] - feature_summary['std']*4]
skew = pd.concat([right_skew.T, left_skew.T], axis=0, join='outer')
skew.head(8)<jupyter_output><empty_output><jupyter_text>No major outliers, just skewed data<jupyter_code>fig, axes = plt.subplots(2, 6, figsize=(14, 5))
fig.suptitle('Skewed Features - Boxplots')
for i,j in zip(skew.columns, range(11)):
    sns.boxplot(ax = axes[int(j/6), j%6], x = df[i])

fig.tight_layout()<jupyter_output><empty_output><jupyter_text>Modeling Prep<jupyter_code>class BaseImputer:
    #@abstractmethod
    def fit(self, X, y=None):
        pass
    
    #@abstractmethod
    def transform(self, X):
        pass

class BaseModel:
    #@abstractmethod
    def fit(self, X, y, sample_weight=None):
        pass
    
    #@abstractmethod
    def predict(self, X):
        pass
class Modeling:
    _X_train_fitted = None
    _X_test_fitted = None
    _y_train = None
    _y_test = None
    _y_preds = None
    
    def __init__(self, data: pd.DataFrame, 
                 target_name: str, 
                 shuffle_splitter: BaseShuffleSplit, 
                 imputer: BaseImputer, 
                 model: BaseModel, 
                 scaler = None):
        self._data = data
        self._target_name = target_name
        self._shuffle_splitter = shuffle_splitter
        self._imputer = imputer
        self._model = model
        self._X, self._y = self._split_data()
        self._scaler = scaler
        
    @property
    def X(self):
        return self._X
    
    @property
    def y(self):
        return self._y

    @property
    def model(self):
        return self._model
    
    @model.setter
    def model(self, model):
        self._model = model
     
    @property
    def X_train(self):
        return self._X_train_fitted
    
    @property
    def X_test(self):
        return self._X_test_fitted
    
    @property
    def y_train(self):
        return self._y_train
    
    @property
    def y_test(self):
        return self._y_test
    
    @property
    def y_preds(self):
        return self._y_preds
    
    def _split_data(self):
        X = self._data.copy()
        return X.drop([self._target_name], axis=1) , X[self._target_name]
    
    def _shuffle_split(self):
        X = self.X
        y = self.y
        for train_index, test_index in self._shuffle_splitter.split(X,y):
            X_train, X_test = X.iloc[train_index], X.iloc[test_index]
            y_train, y_test = y[train_index], y[test_index]
        return X_train, X_test, y_train, y_test
    
    def _fit_imputer(self, train):
        if self._imputer is not None:
            self._imputer.fit(train)
    
    def _fit_scaler(self, train):
        if self._scaler is not None:
            self._scaler.fit(train)
    
    def _impute_data(self, X: pd.DataFrame):
        if self._imputer is not None:
            return pd.DataFrame(self._imputer.transform(X), columns = self.X.columns, index = X.index)
        return X
    
    def _scale_data(self, X: pd.DataFrame):
        if self._scaler is not None:
            X = pd.DataFrame(self._scaler.transform(X), columns = self._X.columns)
        return X
    
    def prepare(self):
        X_train, X_test, y_train, y_test = self._shuffle_split()   
        self._fit_imputer(X_train)
        X_train = self._impute_data(X_train)
        X_test = self._impute_data(X_test)
        self._fit_scaler(X_train)
        self._X_train_fitted = self._scale_data(X_train)
        self._X_test_fitted = self._scale_data(X_test)
        self._y_train = y_train
        self._y_test = y_test
        
    def prepare_and_train(self):
        self.prepare()
        return self.train()
        
    def train(self): #, epoch=None, batch=None
        self._model.fit(self.X_train, self.y_train) #, batch_size=batch, epochs=epoch
        self._y_preds = self._model.predict(self.X_train)
        
        return self.metrics(self.y_train, self.y_preds)
        
    def test(self):
        return self.metrics(self.y_test, self._model.predict(self.X_test))
       
        
    def metrics(self, y_true = None, y_pred = None):
        pass
class ClassificationModeling(Modeling):
    def __init__(self, 
                 data: pd.DataFrame, 
                 target_name: str, 
                 shuffle_splitter: BaseShuffleSplit, 
                 imputer: BaseImputer, 
                 model: BaseModel, 
                 scaler = None,
                 beta: int = 1,
                 classification: str = 'binary'):
        super().__init__(data, target_name, shuffle_splitter, imputer, model, scaler)
        self.beta = beta
        self.classification = classification
    
    def metrics(self, y_true = None, y_pred = None):
        if y_true is None and y_pred is None:
            y_true = self.y_train
            y_pred = self.y_preds
        return ({'matrix': confusion_matrix(y_true, y_pred), 
            'accuracy': accuracy_score(y_true, y_pred), 
            'precision': precision_score(y_true, y_pred, average=self.classification), 
            'recall': recall_score(y_true, y_pred, average=self.classification),
             'f1': f1_score(y_true, y_pred),
            'f{}'.format(self.beta) : fbeta_score(y_true, y_pred, average=self.classification, beta=self.beta) } )
class NNClassificationModeling(ClassificationModeling):
    def __init__(self, 
             data: pd.DataFrame, 
             target_name: str, 
             shuffle_splitter: BaseShuffleSplit, 
             imputer: BaseImputer, 
             model: BaseModel, 
             scaler = None,
             beta: int = 1,
             classification: str = 'binary', tb_callback = TensorBoard(log_dir="logs/", histogram_freq=1)):
        super().__init__(data, target_name, shuffle_splitter, imputer, model, scaler, beta, classification)
        self.tb_callback=tb_callback
        
        
    def train(self, epoch, batch):
        logDir = "logs/{epoch}-{batchsize}-{time}".format(epoch=epoch, batchsize=batch, time=time.time())
        self.tb_callback.log_dir = logDir
        self._model.fit(self.X_train, self.y_train, batch_size=batch, epochs=epoch, validation_data=(self.X_test, self.y_test), callbacks=[self.tb_callback])
        self._y_preds = self._model.predict(self.X_train)
        return self.metrics(self.y_train, self.y_preds)
    
    def metrics(self, y_true = None, y_pred = None):
        if y_true is None and y_pred is None:
            y_true = self.y_train
            y_pred = self.y_preds
            
        y_pred = pd.Series(y_pred.reshape((y_pred.shape[1], y_pred.shape[0]))[0], index=y_true.index)
        y_pred = pd.Series( (y_pred>0.5).astype(int), index=y_true.index)
        return super().metrics(y_true,y_pred)<jupyter_output><empty_output><jupyter_text>Baseline Model - Logistic Regression<jupyter_code>baseline = ClassificationModeling(df,'# label',
                           StratifiedShuffleSplit(n_splits=1, test_size=0.3, random_state=12343),
                           None,
                           LogisticRegression(penalty='l1', solver='saga', random_state=12343),
                           StandardScaler(), beta=2)

baseline.prepare()
baseline_results = pd.DataFrame()

for i in [0.0001, 0.0005, 0.001, .005, 1, 100]:
    baseline.model.C = i
    baseline_results = baseline_results.append({"C": i,
                                               "Train Accuracy": round( baseline.train()['accuracy'], 4),
                                               "Test Accuracy": round( baseline.test()['accuracy'], 4)},
                                              ignore_index=True)
sns.lineplot(data=baseline_results, x='C', y='Train Accuracy', color='blue')
sns.lineplot(data=baseline_results, x='C', y='Test Accuracy', color='red')
plt.title('Tuning C for Logistic Regression with L1')
plt.legend(['Train', 'Test'])
plt.xscale('log')
plt.axvline(0.001, color='black', ls='--')
plt.show()
baseline.model.C = 0.001
baseline.train() #epoch=None, batch=None
baseline.test()<jupyter_output><empty_output><jupyter_text>Feature Importance with L1 Regularization<jupyter_code>feat_coef = []
feat = zip(baseline.X_train.columns, baseline.model.coef_[0])
[feat_coef.append([i,j]) for i,j in feat]
feat_coef = pd.DataFrame(feat_coef, columns = ['feature','coef'])
top_feat_baseline = feat_coef.loc[abs(feat_coef['coef'])>0].sort_values(by='coef')

feat_plot = sns.barplot(data=top_feat_baseline, x='feature', y='coef', palette = "ch:s=.25,rot=-.25")
plt.xticks(rotation=90)
plt.title('LR Feature Importance with L1')
plt.show()<jupyter_output><empty_output><jupyter_text>Eliminated Features from L1 Regularization<jupyter_code>list( feat_coef.loc[feat_coef['coef']==0, 'feature'] )<jupyter_output><empty_output><jupyter_text>See if ElasticNet helps with overfitting  features 'mass' and 'f6'<jupyter_code>baseline.model = LogisticRegression(penalty='elasticnet', l1_ratio=0.5, C=0.001, solver='saga', random_state=12343)
print( 'ElasticNet Training Accuracy =', round( baseline.prepare_and_train()['accuracy'], 4) )
print( 'ElasticNet Test Accuracy =', round( baseline.test()['accuracy'], 4) )<jupyter_output>ElasticNet Training Accuracy = 0.8367
ElasticNet Test Accuracy = 0.8368<jupyter_text>Features 'mass' and 'f6' are most important even with some L2 regularization<jupyter_code>feat_coef = []
feat = zip(baseline.X_train.columns, baseline.model.coef_[0])
[feat_coef.append([i,j]) for i,j in feat]
feat_coef = pd.DataFrame(feat_coef, columns = ['feature','coef'])
top_feat_baseline = feat_coef.loc[abs(feat_coef['coef'])>0].sort_values(by='coef')

feat_plot = sns.barplot(data=top_feat_baseline, x='feature', y='coef', palette = "ch:s=.25,rot=-.25")
plt.xticks(rotation=90)
plt.title('LR Feature Importance with ElasticNet')
plt.show()
list( feat_coef.loc[feat_coef['coef']==0, 'feature'] )<jupyter_output><empty_output><jupyter_text>Neural Network Modeling<jupyter_code>NN = NNClassificationModeling(df,'# label',
                           StratifiedShuffleSplit(n_splits=1, test_size=0.3, random_state=12343),
                           None,
                           None,
                           StandardScaler(), beta=2)

NN.prepare()
NN.model = tf.keras.Sequential() # model object

NN.model.add( tf.keras.layers.Input( shape=(NN.X_train.shape[1],) ) )
# specify data shape for first input layer
# columns (features) only, # rows specified by batch size later in fit() 

NN.model.add( tf.keras.layers.Dense(200, activation = 'relu') )
# add these layers sequentially with decreasing # neurons
NN.model.add( tf.keras.layers.Dense(50, activation = 'relu') )

NN.model.add( tf.keras.layers.Dense(1, activation = 'sigmoid') )
# Final layer, Regression Output
# For Classification, use activation = 'sigmoid' or 'softmax' for Final layer

NN.model.compile(optimizer='adam', loss='BinaryCrossentropy', metrics=['accuracy'])
# Have to compile model after specifying layers

NN.train(batch = 100000, epoch=40)
NN.test()
%load_ext tensorboard
%tensorboard --logdir logs<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12213023811555859, 'fraction_numerical': 0.012545873517111889, 'mean_word_length': 2.988427501701838, 'pattern_counts': {'":': 3, '<': 45, '<?xml version=': 0, '>': 49, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '76582', 'license': 'MIT', 'n_tokens_mistral': 4056, 'n_tokens_neox': 3876, 'n_words': 757, 'path': 'ds7333_case_study_6/case6_AL.ipynb', 'repo_name': 'olmosjorge28/QTW-SPRING-2022', 'size': '11717'}