<commit_before>/*
	Tools for extracting tags from text (descriptions, README's),
    cleaning tags, stripping punctuation, and matching.
*/


/*
    Returns words striped of punctuation from input string.
    Split on several types of delimeters, including spaces, hyphens, and colons.
*/
var natural = require('natural');
    stopwords = require('./stopwords.js');

var tokenizer = new natural.WordTokenizer();

var NGrams = natural.NGrams;

var exclusions = stopwords.words;

function _extract(text) {
    var preTags = tokenizer.tokenize(text.toLowerCase());
    var tags = preTags.filter(function(t) {
        return exclusions.indexOf(t) < 0;
    });
    return tags;
}

function _nGrams(array, n, callback) {
    var results = NGrams.ngrams(array, n);
    callback(results);
}

module.exports = {
    extract: _extract,
    nGrams: _nGrams
};
<commit_msg>use wordpunct tokenizer<commit_after>/*
	Tools for extracting tags from text (descriptions, README's),
    cleaning tags, stripping punctuation, and matching.
*/


/*
    Returns words striped of punctuation from input string.
    Split on several types of delimeters, including spaces, hyphens, and colons.
*/
var natural = require('natural');
    stopwords = require('./stopwords.js');

var tokenizer = new natural.WordPunctTokenizer();

var NGrams = natural.NGrams;

var exclusions = stopwords.words;

function _extract(text) {
    var preTags = tokenizer.tokenize(text.toLowerCase());
    var tags = preTags.filter(function(t) {
        return exclusions.indexOf(t) < 0;
    });
    return tags;
}

function _nGrams(array, n, callback) {
    var results = NGrams.ngrams(array, n);
    callback(results);
}

module.exports = {
    extract: _extract,
    nGrams: _nGrams
};

{'fraction_non_alphanumeric': 0.11514104778353483, 'fraction_numerical': 0.0011514104778353484, 'mean_word_length': 3.7616438356164386, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4283063', 'n_tokens_mistral': 560, 'n_tokens_neox': 539, 'n_words': 154}