<commit_before>/**
 * @class FileArchiver
 *
 * @author: <EMAIL>
 * @created: 2/26/15 3:11 PM
 */
var util = require('util' ),
    events = require( 'events' ),
    dash = require( 'lodash' ),
    path = require( 'path' ),
    TreeWalker = require( './TreeWalker' );

var FileArchiver = function(options) {
    'use strict';

    var archiver = this,
        log = options.log,
        fs = options.fs || require( 'fs' ),
        PROGRESS_EVENT = 'archiverProgressEvent',
        COMPLETE_EVENT = 'archiverCompleteEvent';

    events.EventEmitter.call( this );

    this.purge = function(config, callback) {
        log.info( 'purge config: ', config );

        var days = 180,
            walker = new TreeWalker( { log:log } ),
            removeFiles,
            dir,
            folders,
            loop,
            older;

        if (!config) {
            throw new Error('purge requires a configuration spec');
        }

        if (!config.folders) {
            throw new Error('purge config requires a folders spec');
        }

        if (config.olderThanDays) {
            days = config.olderThanDays;
        }

        folders = dash.clone( config.folders );
        older = new Date( Date.now() - days * 1000 * 60 * 60 * 24 );
        log.info('purge files older than: ', older);

        dir = config.cwd || process.env.HOME;

        removeFiles = function(files, callback) {
            var stats;

            while (files.length > 0) {
                var file = files.shift(),
                    stats = fs.statSync( file );

                log.info('purge file: ', file);
                fs.unlinkSync( file );

                archiver.emit( PROGRESS_EVENT, file );
            }

        };

        loop = function() {
            var folder = folders.pop();

            if (folder) {
                walker.findOlder( path.join( dir, folder ), older, function(err, files) {
                    if (err) {
                        log.error( err );
                    } else {
                        log.info('purge files: ', files);<commit_msg>fixed jshint error<commit_after>/**
 * @class FileArchiver
 *
 * @author: <EMAIL>
 * @created: 2/26/15 3:11 PM
 */
var util = require('util' ),
    events = require( 'events' ),
    dash = require( 'lodash' ),
    path = require( 'path' ),
    TreeWalker = require( './TreeWalker' );

var FileArchiver = function(options) {
    'use strict';

    var archiver = this,
        log = options.log,
        fs = options.fs || require( 'fs' ),
        PROGRESS_EVENT = 'archiverProgressEvent',
        COMPLETE_EVENT = 'archiverCompleteEvent';

    events.EventEmitter.call( this );

    this.purge = function(config, callback) {
        log.info( 'purge config: ', config );

        var days = 180,
            walker = new TreeWalker( { log:log } ),
            removeFiles,
            dir,
            folders,
            loop,
            older;

        if (!config) {
            throw new Error('purge requires a configuration spec');
        }

        if (!config.folders) {
            throw new Error('purge config requires a folders spec');
        }

        if (config.olderThanDays) {
            days = config.olderThanDays;
        }

        folders = dash.clone( config.folders );
        older = new Date( Date.now() - days * 1000 * 60 * 60 * 24 );
        log.info('purge files older than: ', older);

        dir = config.cwd || process.env.HOME;

        removeFiles = function(files, callback) {
            var file,
                stats;

            while (files.length > 0) {
                file = files.shift();
                stats = fs.statSync( file );

                log.info('purge file: ', file);
                fs.unlinkSync( file );

                archiver.emit( PROGRESS_EVENT, file );
            }

        };

        loop = function() {
            var folder = folders.pop();

            if (folder) {
                walker.findOlder( path.join( dir, folder ), older, function(err, files) {
                    if (err) {
                        log.error( err );
                    } else {
                        log.info('purge files: ', files);
{'fraction_non_alphanumeric': 0.12455175711212049, 'fraction_numerical': 0.010518766435572556, 'mean_word_length': 1.5250452625226312, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3905590', 'n_tokens_mistral': 1249, 'n_tokens_neox': 1141, 'n_words': 301}