<commit_before>import re
import string

import numpy as np
import pandas as pd
from bs4 import BeautifulSoup

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from utils import happyfuntokenizing

from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS

def clean_up(df, col_names, min_words = 5):
    '''
    Input : data frame and list of columns to clean up
    Returns: cleaned data frame (overwrites those columns)
    Drops user if any essay has < min_words number of words (default = 5)
    '''
    for c in col_names:
        df[c] = df[c].replace(np.nan, '' , regex=True) \
                    .apply(lambda x: BeautifulSoup(x).getText().replace('\n', ' '))\
                    .replace('\n', ' ')\
                    .apply(lambda x: re.sub('\s+', ' ', x).strip())
        token_count = df[c].str.split().str.len() 
        df = df[token_count > min_words] #drop rows where current essay has < min_words
    return df<commit_msg>fillna moved to clean_up<commit_after>import re
import string

import numpy as np
import pandas as pd
from bs4 import BeautifulSoup

from sklearn.feature_extraction.text import CountVectorizer, TfidfTransformer
from utils import happyfuntokenizing

from sklearn.feature_extraction.stop_words import ENGLISH_STOP_WORDS

def clean_up(df, col_names, min_words = 5):
    '''
    Input : data frame and list of columns to clean up
    Returns: cleaned data frame (overwrites those columns)
    Drops user if any essay has < min_words number of words (default = 5)
    '''
    for c in col_names:
        df[c] = df[c].replace(np.nan, '' , regex=True) \
                    .apply(lambda x: BeautifulSoup(x).getText().replace('\n', ' '))\
                    .replace('\n', ' ')\
                    .apply(lambda x: re.sub('\s+', ' ', x).strip())
        token_count = df[c].str.split().str.len() 
        df = df[token_count > min_words] #drop rows where current essay has < min_words

        df.fillna('', inplace=True)
    return df
{'fraction_non_alphanumeric': 0.11072492552135055, 'fraction_numerical': 0.0029791459781529296, 'mean_word_length': 2.88996138996139, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2924860', 'n_tokens_mistral': 627, 'n_tokens_neox': 614, 'n_words': 188}