<commit_before>import re

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import regexp_tokenize


def split_on_sentence(text):
    """Tokenize the text on sentences.
    Returns a list of strings (sentences).
    """
    sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
    return sent_tokenizer.tokenize(text)

def re_punc(text):
    """Remove all punctuation. Keep apostrophes."""
    return re.sub(r'[!"#$%&()*+,\-\./:;<=>?@\[\]^_`\\{\|}]+', ' ', text)

def remove_punctuation(sentences):
    """Remove punctuation based on `re_punc`.
    Returns either a list of string or a single string,
    based on the input type.
    """
    if type(sentences) is list:
        return [re_punc(sentence).strip() for sentence in sentences]
    else:
        return re_punc(sentences).strip()

def split_on_word(text):
    """Use regular expression tokenizer.
    Keep apostrophes.
    Returns a list of lists, one list for each sentence:
        [[word, word], [word, word, ..., word], ...].
    """
    if type(text) is list:
        return [regexp_tokenize(sentence, pattern="\w+(?:[-']\w+)*")
                for sentence in text]
    else:
        return regexp_tokenize(text, pattern="\w+(?:[-']\w+)*")

def normalize(tokenized_words):
    """Removes stop words, numbers, short words, and lowercases text.
    Returns a list of lists, one list for each sentence:
        [[word, word], [word, word, ..., word], ...].
    """
    stop_words = stopwords.words('english')
    return [[w.lower() for w in sent
             if (w.lower() not in stop_words) and\
             (not(w.lower().isnumeric())) and\
             (len(w) > 2)]
            for sent in tokenized_words]<commit_msg>make stopword removal optional<commit_after>import re

import nltk
from nltk.corpus import stopwords
from nltk.tokenize import regexp_tokenize


def split_on_sentence(text):
    """Tokenize the text on sentences.
    Returns a list of strings (sentences).
    """
    sent_tokenizer = nltk.data.load('tokenizers/punkt/english.pickle')
    return sent_tokenizer.tokenize(text)

def re_punc(text):
    """Remove all punctuation. Keep apostrophes."""
    return re.sub(r'[!"#$%&()*+,\-\./:;<=>?@\[\]^_`\\{\|}]+', ' ', text)

def remove_punctuation(sentences):
    """Remove punctuation based on `re_punc`.
    Returns either a list of string or a single string,
    based on the input type.
    """
    if type(sentences) is list:
        return [re_punc(sentence).strip() for sentence in sentences]
    else:
        return re_punc(sentences).strip()

def split_on_word(text):
    """Use regular expression tokenizer.
    Keep apostrophes.
    Returns a list of lists, one list for each sentence:
        [[word, word], [word, word, ..., word], ...].
    """
    if type(text) is list:
        return [regexp_tokenize(sentence, pattern="\w+(?:[-']\w+)*")
                for sentence in text]
    else:
        return regexp_tokenize(text, pattern="\w+(?:[-']\w+)*")

def normalize(tokenized_words, remove_stopwords = True):
    """Removes stop words, numbers, short words, and lowercases text.
    Returns a list of lists, one list for each sentence:
        [[word, word], [word, word, ..., word], ...].
    """
    if remove_stopwords:
        stop_words = stopwords.words('english')
        return [[w.lower() for w in sent
                 if (w.lower() not in stop_words) and\
                 (not(w.lower().isnumeric())) and\
                 (len(w) > 2)]
                for sent in tokenized_words]
    else:
        return [[w.lower() for w in sent
                if not(w.lower().isnumeric())]
                for sent in tokenized_words]
{'fraction_non_alphanumeric': 0.1492089470812875, 'fraction_numerical': 0.0005455537370430987, 'mean_word_length': 2.7003027245206863, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4905097', 'n_tokens_mistral': 1173, 'n_tokens_neox': 1141, 'n_words': 339}