<commit_before>import fasttext
import gensim
import numpy as np
import pandas as pd
import re


EMOJIS = pd.read_csv('data/emojis.txt', header=None)[0].tolist()

SPAMMY_PATTERNS = [
    'bbm|pin bb|line|whatsapp',
    '(ch?ec?k|intip|liat|kepoin)( out)?( (our|my))? (ig|insta|koleksi|profil)',
    'cekidot',
    'dada|herba|langsing|payudara|pemutih|penggemuk|peninggi|tahan lama|tinggi badan',
    'efek samping',
    'follow',
    'free (delivery|ongkos|ongkir|pengiriman)|gratis|murah|promo|terjangkau',
    'garansi|kualitas',
    'impor',
    'invit',
    'jerawat',
    'jual|sell',
    'langganan',
    'luar biasa',
    'mampir',
    'nyaranin',
    'order',
    'password',
    'penghasilan',
    'produk',
    's(o|u|ou)venir',
    'stock|stok',
    'yu+k',
]


def count_numbers(texts):
    # type: (pd.Series) -> pd.Series
    """ Return number of digits for each text in texts """
    return texts.apply(lambda x: sum(c.isdigit() for c in x))

def count_capitals(texts):
    # type: (pd.Series) -> pd.Series
    """ Return number of capital letters for each text in texts """
    return texts.apply(lambda x: sum(c.isupper() for c in x))

def count_emojis(texts):
    # type: (pd.Series) -> pd.Series
    """ Return number of emojis for each text in texts """
    return texts.apply(lambda x: sum(c in EMOJIS for c in x))

def extract_emojis(texts):
    # type: (pd.Series) -> pd.Series
    """ Return all emojis contained in each text in texts """
    return texts.apply(lambda x: [c for c in x if c in EMOJIS])

def extract_spammy_pattern_features(texts):
    # type: (pd.Series) -> pd.DataFrame
    """ Return a DataFrame for each pattern in SPAMMY_PATTERNS,
    denoting whether each text in text contains the pattern """
    spammy_pattern_features = {
        'has_pattern_{}'.format(pat): texts.str.contains(pat.replace(' +', '\s+'), case=False)
        for pat in SPAMMY_PATTERNS
    }
    spammy_pattern_features = pd.DataFrame(spammy_pattern_features)
    spammy_pattern_features.index = texts.index
    return spammy_pattern_features

def tokenize(texts):
    # type: (pd.Series) -> pd.Series
    """ Return tokenization (as list of str) of each text in texts """
    return texts.apply(lambda x: re.findall(r'\b\w+\b', x.lower()))

def tokenize_2(texts):
    # type: (pd.Series) -> pd.Series
    """ Return tokenization (as list of str) of each text in texts.
        Use a different regex than tokenize()
    """
    re_tokenize = re.compile('[!"#$%&\'()*+,./:;<=>?@\[\]^_{|}~\s-]')
    return texts\
        .apply(re_tokenize.split)\
        .apply(lambda x: [tok for tok in x if tok])    # remove empty tokens

def extract_word_embedding(tokens):
    # type: (pd.Series) -> pd.DataFrame
    """ Return a DataFrame for 100-dimension word embedding """
    word_embedding = pd.DataFrame(_get_w2v_mean(tokens).tolist(), columns=['embedding_{}'.format(i) for i in range(100)])
    word_embedding.index = tokens.index
    return word_embedding

def _get_w2v_mean(tokens, fpath_model='data/classification.model'):
    model = gensim.models.Word2Vec.load(fpath_model)
    return tokens.apply(lambda x: _get_mean(x, model))

def extract_fasttext(texts):
    # type: (pd.Series) -> pd.DataFrame
    """ Return a DataFrame for 100-dimension word embedding """
    tokens = tokenize(texts)
    embedding = pd.DataFrame(_get_fasttext_mean(tokens).tolist(), columns=['embedding_{}'.format(i) for i in range(100)])
    embedding.index = texts.index
    return embedding

def train_model_fasttext(tokens, file_in='data/train.txt', path_out='/tmp/model'):
    tokens.apply(' '.join).to_csv(file_in, index=False) # writing to .txt file
    model = fasttext.skipgram(file_in, path_out)
    return model

def _get_fasttext_mean(tokens, fpath_model='/tmp/model.bin'):
    model = fasttext.load_model(fpath_model)
    return tokens.apply(lambda x: _get_mean(x, model))

def _get_mean(tokens, model):
    i = 0<commit_msg>Added file path arg to extract_fasttext()<commit_after>import fasttext
import gensim
import numpy as np
import pandas as pd
import re


EMOJIS = pd.read_csv('data/emojis.txt', header=None)[0].tolist()

SPAMMY_PATTERNS = [
    'bbm|pin bb|line|whatsapp',
    '(ch?ec?k|intip|liat|kepoin)( out)?( (our|my))? (ig|insta|koleksi|profil)',
    'cekidot',
    'dada|herba|langsing|payudara|pemutih|penggemuk|peninggi|tahan lama|tinggi badan',
    'efek samping',
    'follow',
    'free (delivery|ongkos|ongkir|pengiriman)|gratis|murah|promo|terjangkau',
    'garansi|kualitas',
    'impor',
    'invit',
    'jerawat',
    'jual|sell',
    'langganan',
    'luar biasa',
    'mampir',
    'nyaranin',
    'order',
    'password',
    'penghasilan',
    'produk',
    's(o|u|ou)venir',
    'stock|stok',
    'yu+k',
]


def count_numbers(texts):
    # type: (pd.Series) -> pd.Series
    """ Return number of digits for each text in texts """
    return texts.apply(lambda x: sum(c.isdigit() for c in x))

def count_capitals(texts):
    # type: (pd.Series) -> pd.Series
    """ Return number of capital letters for each text in texts """
    return texts.apply(lambda x: sum(c.isupper() for c in x))

def count_emojis(texts):
    # type: (pd.Series) -> pd.Series
    """ Return number of emojis for each text in texts """
    return texts.apply(lambda x: sum(c in EMOJIS for c in x))

def extract_emojis(texts):
    # type: (pd.Series) -> pd.Series
    """ Return all emojis contained in each text in texts """
    return texts.apply(lambda x: [c for c in x if c in EMOJIS])

def extract_spammy_pattern_features(texts):
    # type: (pd.Series) -> pd.DataFrame
    """ Return a DataFrame for each pattern in SPAMMY_PATTERNS,
    denoting whether each text in text contains the pattern """
    spammy_pattern_features = {
        'has_pattern_{}'.format(pat): texts.str.contains(pat.replace(' +', '\s+'), case=False)
        for pat in SPAMMY_PATTERNS
    }
    spammy_pattern_features = pd.DataFrame(spammy_pattern_features)
    spammy_pattern_features.index = texts.index
    return spammy_pattern_features

def tokenize(texts):
    # type: (pd.Series) -> pd.Series
    """ Return tokenization (as list of str) of each text in texts """
    return texts.apply(lambda x: re.findall(r'\b\w+\b', x.lower()))

def tokenize_2(texts):
    # type: (pd.Series) -> pd.Series
    """ Return tokenization (as list of str) of each text in texts.
        Use a different regex than tokenize()
    """
    re_tokenize = re.compile('[!"#$%&\'()*+,./:;<=>?@\[\]^_{|}~\s-]')
    return texts\
        .apply(re_tokenize.split)\
        .apply(lambda x: [tok for tok in x if tok])    # remove empty tokens

def extract_word_embedding(tokens):
    # type: (pd.Series) -> pd.DataFrame
    """ Return a DataFrame for 100-dimension word embedding """
    word_embedding = pd.DataFrame(_get_w2v_mean(tokens).tolist(), columns=['embedding_{}'.format(i) for i in range(100)])
    word_embedding.index = tokens.index
    return word_embedding

def _get_w2v_mean(tokens, fpath_model='data/classification.model'):
    model = gensim.models.Word2Vec.load(fpath_model)
    return tokens.apply(lambda x: _get_mean(x, model))

def extract_fasttext(texts, fpath_model='/tmp/model.bin'):
    # type: (pd.Series) -> pd.DataFrame
    """ Return a DataFrame for 100-dimension word embedding """
    tokens = tokenize(texts)
    embedding = pd.DataFrame(_get_fasttext_mean(tokens, fpath_model=fpath_model).tolist(),
                             columns=['embedding_{}'.format(i) for i in range(100)])
    embedding.index = texts.index
    return embedding

def train_model_fasttext(tokens, file_in='data/train.txt', path_out='/tmp/model'):
    tokens.apply(' '.join).to_csv(file_in, index=False) # writing to .txt file
    model = fasttext.skipgram(file_in, path_out)
    return model

def _get_fasttext_mean(tokens, fpath_model='/tmp/model.bin'):
    model = fasttext.load_model(fpath_model)
    return tokens.apply(lambda x: _get_mean(x, model))

def _get_mean(tokens, model):
    i = 0
{'fraction_non_alphanumeric': 0.1448820873055695, 'fraction_numerical': 0.004515805318615153, 'mean_word_length': 3.717751479289941, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 23, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1249174', 'n_tokens_mistral': 2893, 'n_tokens_neox': 2822, 'n_words': 668}