<commit_before>'use strict';

var _ = require('lodash'),
    Walk = require('walk'),
    Path = require('path');

function sources(source, done) {
    var walker = Walk.walk(source),
        collect = [];

    walker.on('file', function(root, stats, next) {
        collect.push(Path.join(root, stats.name));
        next();
    });

    walker.on('end', function() {
        done(null, collect);
    });
}

function scan(config, done) {
    sources(config.source, function(err, files) {
        var re = new RegExp('^' + config.source);

        config.sources = _.map(files, function(file) {
            return file.replace(re, '');
        });
        done();
    });
}

module.exports = scan;<commit_msg>remove fiddly bit<commit_after>'use strict';

var Walk = require('walk'),
    Path = require('path');

function sources(source, done) {
    var walker = Walk.walk(source),
        collect = [];

    walker.on('file', function(root, stats, next) {
        collect.push(Path.join(root, stats.name));
        next();
    });

    walker.on('end', function() {
        done(null, collect);
    });
}

function scan(config, done) {
    sources(config.source, function(err, files) {

        config.sources = files;

        done();
    });
}

module.exports = scan;
{'fraction_non_alphanumeric': 0.1750788643533123, 'fraction_numerical': 0.0, 'mean_word_length': 2.6153846153846154, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1715739', 'n_tokens_mistral': 402, 'n_tokens_neox': 388, 'n_words': 82}