<commit_before>import re

import numpy as np

import nltk

__all__ = ['strip_punc', 'rem_num', 'rehyph', 'add_metadata',
           'apply_stoplist', 'filter_by_suffix', 'word_tokenize',
           'sentence_tokenize', 'paragraph_tokenize']



def strip_punc(tsent):
    """
    """
    p1 = re.compile(r'^(\W*)')
    p2 = re.compile(r'(\W*)$')

    out = []
    for word in tsent:
        w = re.sub(p2, '', re.sub(p1, '', word))
        if w:
            out.append(w)

    return out


def rem_num(tsent):
    """
    """
    p = re.compile(r'(^\D+$)|(^\D*[1-2]\d\D*$|^\D*\d\D*$)')

    return [word for word in tsent if re.search(p, word)]


def rehyph(sent):
    """
    """
    return re.sub(r'(?P<x1>.)--(?P<x2>.)', '\g<x1> - \g<x2>', sent)


def add_metadata(corpus, ctx_type, new_field, metadata):
    """
    Returns a corpus with metadata added.

    :param corpus: Corpus object to add new metadata to.
    :type corpus: Corpus

    :param ctx_type: A type of tokenization.
    :type ctx_type: string

    :param new_field: Field name of the new metadata.
    :type new_field: string

    :param metadata: List of values to be added to `corpus`.
    :type metdata: list

    :returns: Corpus with new metadata added to the existing metdata.

    :See Also: :class:`vsm.corpus.Corpus`
    """
    from vsm import arr_add_field
    i = corpus.context_types.index(ctx_type)
    md = corpus.context_data[i]
    corpus.context_data[i] = arr_add_field(md, new_field, metadata)

    return corpus


def change_label(c_path, oldlabel, newlabel, new_path=''):
    """
    modify metadata label in context_data.
    """
    from vsm.corpus import Corpus
    
    if not new_path:
        new_path  = c_path
        
    c = Corpus.load(c_path)
    for ctx in c.context_data:
        names = [a  if a != oldlabel else newlabel for a in ctx.dtype.names]
        tup = tuple(names)
        ctx.dtype.names = tup   
    
    c.save(new_path)


def apply_stoplist(corp, nltk_stop=True, add_stop=None, freq=0):
    """
    Returns a Corpus object with stop words eliminated.

    :param corp: Corpus object to apply stoplist to.<commit_msg>added corpussent option to change_label.<commit_after>import re

import numpy as np

import nltk

__all__ = ['strip_punc', 'rem_num', 'rehyph', 'add_metadata',
           'apply_stoplist', 'filter_by_suffix', 'word_tokenize',
           'sentence_tokenize', 'paragraph_tokenize']



def strip_punc(tsent):
    """
    """
    p1 = re.compile(r'^(\W*)')
    p2 = re.compile(r'(\W*)$')

    out = []
    for word in tsent:
        w = re.sub(p2, '', re.sub(p1, '', word))
        if w:
            out.append(w)

    return out


def rem_num(tsent):
    """
    """
    p = re.compile(r'(^\D+$)|(^\D*[1-2]\d\D*$|^\D*\d\D*$)')

    return [word for word in tsent if re.search(p, word)]


def rehyph(sent):
    """
    """
    return re.sub(r'(?P<x1>.)--(?P<x2>.)', '\g<x1> - \g<x2>', sent)


def add_metadata(corpus, ctx_type, new_field, metadata):
    """
    Returns a corpus with metadata added.

    :param corpus: Corpus object to add new metadata to.
    :type corpus: Corpus

    :param ctx_type: A type of tokenization.
    :type ctx_type: string

    :param new_field: Field name of the new metadata.
    :type new_field: string

    :param metadata: List of values to be added to `corpus`.
    :type metdata: list

    :returns: Corpus with new metadata added to the existing metdata.

    :See Also: :class:`vsm.corpus.Corpus`
    """
    from vsm import arr_add_field
    i = corpus.context_types.index(ctx_type)
    md = corpus.context_data[i]
    corpus.context_data[i] = arr_add_field(md, new_field, metadata)

    return corpus


def change_label(c_path, oldlabel, newlabel, new_path='', corpussent=False):
    """
    modify metadata label in context_data.
    """
    from vsm.corpus import Corpus
    from vsm.extensions.ldasentences import CorpusSent

    if not new_path:
        new_path  = c_path
        
    c = Corpus.load(c_path)
    if corpussent:
        c = CorpusSent.load(c_path)
        
    for ctx in c.context_data:
        names = [a  if a != oldlabel else newlabel for a in ctx.dtype.names]
        tup = tuple(names)
        ctx.dtype.names = tup   
    
    c.save(new_path)


def apply_stoplist(corp, nltk_stop=True, add_stop=None, freq=0):
    """
    Returns a Corpus object with stop words eliminated.

    :param corp: Corpus object to apply stoplist to.
{'fraction_non_alphanumeric': 0.13447653429602888, 'fraction_numerical': 0.004963898916967509, 'mean_word_length': 2.6636363636363636, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4483820', 'n_tokens_mistral': 1629, 'n_tokens_neox': 1572, 'n_words': 418}