<commit_before>var _ = require('underscore');
var vm = require('vm');
var falafel = require('falafel');
var gettext = require('../gettext');


function transform(src, opts, fn) {
    if (typeof opts == 'function')  {
        fn = opts;
        opts = {};
    }

    opts = _.extend({ecmaVersion: 6}, opts);
    return falafel(src, opts, fn);
}


function extract(opts) {
    var src = yoink_members(opts.source, opts);

    return gather(src, opts).map(function(node) {
        return run(node, opts);
    });
}


function yoink_members(src, opts) {
    opts = _.extend({}, opts);

    return transform(src, opts, function(node) {
        if (is_member(node, opts)) {
            node.parent.update(node.source());
        }
    }).toString();
}


function gather(src, opts) {
    var matches = [];

    opts = _.extend({}, { locations: true }, opts);

    transform(src, opts, function(node) {
        if (match(node, opts)) {<commit_msg>Isolating all parser options into a key to prevent possible future naming conflicts.<commit_after>var _ = require('underscore');
var vm = require('vm');
var falafel = require('falafel');
var gettext = require('../gettext');


function transform(src, opts, fn) {
    if (typeof opts == 'function')  {
        fn = opts;
        opts = {};
    }

    opts.parserOptions = _.extend({}, {
        ecmaVersion: 6
    }, opts.parserOptions);

    return falafel(src, opts.parserOptions, fn);
}


function extract(opts) {
    var src = yoink_members(opts.source, opts);

    return gather(src, opts).map(function(node) {
        return run(node, opts);
    });
}


function yoink_members(src, opts) {
    opts = _.extend({}, opts);

    return transform(src, opts, function(node) {
        if (is_member(node, opts)) {
            node.parent.update(node.source());
        }
    }).toString();
}


function gather(src, opts) {
    var matches = [];

    opts.parserOptions = _.extend({}, {
        locations: true
    }, opts.parserOptions);

    transform(src, opts, function(node) {
        if (match(node, opts)) {
{'fraction_non_alphanumeric': 0.16146341463414635, 'fraction_numerical': 0.000975609756097561, 'mean_word_length': 2.848030018761726, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '470586', 'n_tokens_mistral': 658, 'n_tokens_neox': 631, 'n_words': 159}