<jupyter_start><jupyter_text>Scoring Function<jupyter_code>def score_model(sklearn_model, preprocessing = None):
    
    # Store the holdout predictions
    oof_preds = np.zeros((train.shape[0],))
    test_preds = np.zeros((len(test),len(train['target'].unique())))
    scores = np.zeros(NUM_FOLDS)
    times = np.zeros(NUM_FOLDS)
    print('')
    
    # Stratified k-fold cross-validation
    skf = StratifiedKFold(n_splits = NUM_FOLDS, shuffle = True, random_state = RANDOM_SEED)
    for fold, (train_idx, valid_idx) in enumerate(skf.split(train, target_bins)):
        
        # Training and Validation Sets
        X_train, y_train = train[features].iloc[train_idx], train['target'].iloc[train_idx]
        X_valid, y_valid = train[features].iloc[valid_idx], train['target'].iloc[valid_idx]
        train_weight, valid_weight = train['sample_weight'].iloc[train_idx], train['sample_weight'].iloc[valid_idx]
        X_test = test[features]
    
        # Preprocessing
        start = time.time()
        if preprocessing:
            preprocessing.fit(X_train, y_train, train_weight)
            X_train, y_train, train_weight = preprocessing.transform(
                X_train, y_train, train_weight
            )
        gc.collect()
        
        # Create model
        model = clone(sklearn_model)
        try:
            model.fit(X_train, y_train, sample_weight = train_weight)
        except:
            model.fit(X_train, y_train)
        
        # validation
        valid_preds = model.predict(X_valid)
        test_preds += model.predict_proba(X_test) / NUM_FOLDS
        scores[fold] = accuracy_score(y_valid, valid_preds, sample_weight = valid_weight)
        oof_preds[valid_idx] = valid_preds.flatten()
        end = time.time()
        print(f'Fold {fold}: {round(scores[fold], 5)} accuracy in {round(end-start,2)}s.')
        times[fold] = end-start
        gc.collect()
        
    mask1, mask10 = train.gcd == 1, train.gcd == 10 
    mask1000, mask10000 = train.gcd == 1000, train.gcd == 10000
    print("\nAccuracy (1M Reads):", round(accuracy_score(oof_preds[mask1], train['target'].loc[mask1], sample_weight = train['sample_weight'].loc[mask1]), 5))
    print("Accuracy (100k Reads):", round(accuracy_score(oof_preds[mask10], train['target'].loc[mask10], sample_weight = train['sample_weight'].loc[mask10]), 5))
    print("Accuracy (1k Reads):", round(accuracy_score(oof_preds[mask1000], train['target'].loc[mask1000], sample_weight = train['sample_weight'].loc[mask1000]), 5))
    print("Accuracy (100 Reads):", round(accuracy_score(oof_preds[mask10000], train['target'].loc[mask10000], sample_weight = train['sample_weight'].loc[mask10000]), 5))
    print("Out-of-Fold Accuracy:", round(accuracy_score(oof_preds, train['target'], sample_weight = train['sample_weight']), 5))
    print(f'Training Time: {round(times.sum(), 2)}s')
    
    return oof_preds, np.argmax(test_preds, axis = 1)
from sklearn.metrics import ConfusionMatrixDisplay, confusion_matrix, accuracy_score
import matplotlib.pyplot as plt
from math import factorial

def bias_of(column):
    w = int(column[1:column.index('T')])
    x = int(column[column.index('T')+1:column.index('G')])
    y = int(column[column.index('G')+1:column.index('C')])
    z = int(column[column.index('C')+1:])
    return factorial(10) / (factorial(w) * factorial(x) * factorial(y) * factorial(z) * 4**10)


# Plot confusion matrix
def plot_confusion_matrix(true_values, pred_values, gcds, plot_title = "Confusion Matrix"):
    
    gcd = [[1,10],[1000,10000]]
    
    # Confusion matrix
    fig, ax = plt.subplots(2, 2, figsize = (12,9))
    for row in range(2):
        for col in range(2):
            idx = 2*row + col
            cm = confusion_matrix(true_values[gcds == gcd[row][col]], pred_values[gcds == gcd[row][col]])
            np.fill_diagonal(cm, 0)
            disp = ConfusionMatrixDisplay(confusion_matrix = cm)
            disp.plot(ax = ax[row,col])
    plt.show()<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code>%%time
from sklearn.preprocessing import LabelEncoder

def load_data(test = False):
    train = pd.read_feather('../data/train.feather')
    features = [x for x in train.columns if x not in ['row_id','target','sample_weight','gcd']]

    encoder = LabelEncoder()
    train['target'] = encoder.fit_transform(train['target'])
    return train, encoder, features

train, encoder, features = load_data()
target_bins = train['target'].astype(str) + train['gcd'].astype(str)
test = pd.read_feather('../data/test.feather')
submission = pd.read_csv('../data/sample_submission.csv')

print(f'Training Samples: {len(train)}')<jupyter_output>Training Samples: 123993
CPU times: total: 2.12 s
Wall time: 403 ms<jupyter_text>Baseline<jupyter_code>oof_preds, test_preds = score_model(ExtraTreesClassifier(**PARAMS))

plot_confusion_matrix(train['target'],oof_preds, train['gcd'])

submission['target'] = encoder.inverse_transform(test_preds)
submission.to_csv('../submissions/extratrees_baseline.csv', index=False)<jupyter_output>Fold 0: 0.96042 accuracy in 32.04s.
Fold 1: 0.95267 accuracy in 31.93s.
Fold 2: 0.95456 accuracy in 32.19s.
Fold 3: 0.95849 accuracy in 32.26s.
Fold 4: 0.95536 accuracy in 31.86s.
Fold 5: 0.95723 accuracy in 32.0s.
Fold 6: 0.95162 accuracy in 31.82s.
Fold 7: 0.9559 accuracy in 32.15s.
Fold 8: 0.95726 accuracy in 31.96s.
Fold 9: 0.95857 accuracy in 32.12s.

Accuracy (1M Reads): 1.0
Accuracy (100k Reads): 1.0
Accuracy (1k Reads): 0.92183
Accuracy (100 Reads): 0.90308
Out-of-Fold Accuracy: 0.95622
Training Time: 320.32s<jupyter_text>Data Augmentation<jupyter_code>class Augmenter(BaseEstimator, TransformerMixin):
    
    def __init__(self, num_samples, error_rate = 0.05):
        self.num_samples = num_samples
        self.rng = np.random.default_rng()
        self.rate = error_rate
        self.dist = [bias_of(x) for x in features]
        
    def fit(self, X, y, sample_weights):
        
        # Convert to integers
        temp = pd.DataFrame({
            col: ((X[col] + bias_of(col)) * 1000000).round().astype(int) for col in features
        })
        
        # Get GCDs
        gcd = temp[features[0]]
        for col in features[1:]:
            gcd = np.gcd(gcd, temp[col])
        
        # Raw histograms
        for col in features:
            temp[col] //= gcd
            temp[col] *= sample_weights
        
        # Save sample space
        temp['sample_weight'] = sample_weights
        temp['gcd'] = gcd
        temp['target'] = y
        self.samples = dict()
        self.prob = dict()
        for label in np.unique(y):
            self.samples[label] = temp[(temp.gcd > 100) & (temp.target == label)].reset_index(drop=True)
            self.prob[label] = temp[(temp.gcd > 100) & (temp.target == label)]['sample_weight'].to_numpy() / np.sum(temp[(temp.gcd > 100) & (temp.target == label)]['sample_weight'])
        
    
    def transform(self, X, y, sample_weights):
        X = X[features].copy()
        
        indices = list()
        labels = list()
        gcd = list()
        for label in sorted(np.unique(y)):
            # pick indices using sample weights
            temp = self.rng.choice(
                self.samples[label].index, 
                size = self.num_samples,
                p = self.prob[label]
            )
            indices.extend(temp)
            labels.extend([label,]*self.num_samples)
            gcd.extend([self.samples[label]['gcd'].loc[x] for x in temp])
        
        # Data for the new samples
        gcd = [self.samples[label]['gcd'].iloc[x] for x, label in zip(indices,labels)]
        rows = [Counter(self.samples[label][features].iloc[idx].to_dict()) for label, idx in zip(labels,indices)]
        
        # simulate errors
        for sample in rows:
            
            # Number of errors per sample
            num_errors = self.rng.binomial(
                n = sum(sample.values()), 
                p = self.rate
            )
            
            # Reads to replace with errors
            old_reads = self.rng.choice(
                list(sample.elements()), 
                size = num_errors,
                replace = False
            )
            
            # Erroneous reads
            new_reads = self.rng.choice(
                features, 
                size = num_errors,
                p = self.dist
            )
            
            sample.subtract(old_reads)
            sample.update(new_reads)
        
        new_samples = pd.DataFrame.from_records(rows, columns = features).fillna(0).astype(int)
        for col in features:
            new_samples[col] *= gcd
            new_samples[col] /= 1000000
            new_samples[col] -= bias_of(col)
        new_samples['target'] = labels
        new_samples['sample_weight'] = 1
        new_samples = new_samples.sample(frac=1).reset_index(drop=True)
        X = X[features].copy()
        X['sample_weight'] = sample_weights
        X['target'] = y
        new_samples = pd.concat([X, new_samples], ignore_index = True).sample(frac=1).reset_index(drop=True).fillna(0)
        return new_samples[features], new_samples['target'], new_samples['sample_weight']

oof_preds, test_preds = score_model(
    ExtraTreesClassifier(**PARAMS),
    Augmenter(2500)
)

plot_confusion_matrix(train['target'], oof_preds, train['gcd'])  

submission['target'] = encoder.inverse_transform(test_preds)
submission.to_csv('../submissions/augmentation_submission.csv', index=False)<jupyter_output>Fold 0: 0.96022 accuracy in 151.41s.
Fold 1: 0.95577 accuracy in 149.48s.
Fold 2: 0.95617 accuracy in 153.71s.
Fold 3: 0.95923 accuracy in 153.24s.
Fold 4: 0.95621 accuracy in 152.69s.
Fold 5: 0.95913 accuracy in 153.45s.
Fold 6: 0.95415 accuracy in 153.89s.
Fold 7: 0.9572 accuracy in 153.06s.
Fold 8: 0.95538 accuracy in 151.97s.
Fold 9: 0.95923 accuracy in 153.68s.

Accuracy (1M Reads): 1.0
Accuracy (100k Reads): 1.0
Accuracy (1k Reads): 0.92221
Accuracy (100 Reads): 0.90693
Out-of-Fold Accuracy: 0.95728
Training Time: 1526.59s
{'fraction_non_alphanumeric': 0.1332205750671575, 'fraction_numerical': 0.04526912745000498, 'mean_word_length': 2.8498659517426272, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 17, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '399459', 'license': 'MIT', 'n_tokens_mistral': 3735, 'n_tokens_neox': 3321, 'n_words': 697, 'path': 'tps-2022-02/notebooks/Notebook 7 - Data Augmentation.ipynb', 'repo_name': 'rsizem2/tabular-playground-series', 'size': '10051'}