<commit_before>import pandas as pd

train_users = pd.read_csv('../datasets/processed/processed_train_users.csv')
test_users = pd.read_csv('../datasets/processed/processed_test_users.csv')

percentage = 0.95

train_mask = train_users.isnull().sum() > train_users.shape[0] * percentage
train_to_remove = list(train_users.isnull().sum()[train_mask].index)

test_mask = test_users.isnull().sum() > test_users.shape[0] * percentage
test_to_remove = list(test_users.isnull().sum()[test_mask].index)


to_remove = list(set(train_to_remove).intersection(test_to_remove))

train_users.drop(to_remove, axis=1, inplace=True)
test_users.drop(to_remove, axis=1, inplace=True)

train_users.to_csv('clean_processed_train_users.csv')
test_users.to_csv('clean_processed_test_users.csv')<commit_msg>Clean rewrite<commit_after>#!/usr/bin/env python

import pandas as pd


def main():
    path = '../datasets/processed/'
    train_users = pd.read_csv(path + 'processed_train_users.csv')
    test_users = pd.read_csv(path + 'processed_test_users.csv')

    percentage = 0.95

    train_mask = train_users.isnull().sum() > train_users.shape[0] * percentage
    train_to_remove = list(train_users.isnull().sum()[train_mask].index)

    test_mask = test_users.isnull().sum() > test_users.shape[0] * percentage
    test_to_remove = list(test_users.isnull().sum()[test_mask].index)

    to_remove = list(set(train_to_remove).intersection(test_to_remove))

    train_users.drop(to_remove, axis=1, inplace=True)
    test_users.drop(to_remove, axis=1, inplace=True)

    train_users.to_csv('clean_processed_train_users.csv')
    test_users.to_csv('clean_processed_test_users.csv')

if __name__ == '__main__':
    main()
{'fraction_non_alphanumeric': 0.14023668639053255, 'fraction_numerical': 0.008284023668639054, 'mean_word_length': 5.930327868852459, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2473837', 'n_tokens_mistral': 620, 'n_tokens_neox': 593, 'n_words': 76}