<commit_before>


(function(exports) {

    var path = require('path');
    var _ = require('underscore');
    var fs = require('fs');

    function is_in_array(arr,search){
        for( var n in arr ){
            if( arr[n] == search ) return true;
        }
        return false;
    }

    var meta_entry = (function(){
        return function(){
            this._base_dir = ''
            this.reference_time = ''
            this.build = []
            this.dependences = []
            this.extras = {}

            var that = this
            this.save = function(out_path, cb){
                that.update_dependences_addressing()
                that.dedup_dependencies()
                that.update_reference_time()
                var o = {}
                _.extend(o, that)
                delete o._base_dir
                var contents = JSON.stringify(o, null, 4)
                fs.writeFileSync(out_path, contents)
                if( cb ) cb(null)
            }
            this.require_task = function(build_cmd, options){
                var b = {}
                b[build_cmd] = options
                that.build.push(b)
                return that
            }
            this.is_fresh = function(){
                var time = that.reference_time
                var is_fresh = true
                for( var n in that.dependences ){
                    var filename = that.dependences[n]
                    if ( fs.existsSync(filename) ){
                        var ft = fs.statSync(filename);
                        is_fresh = is_fresh && ft.mtime <= time
                    }else{
                        is_fresh = false;
                    }
                }<commit_msg>first commit<commit_after>


(function(exports) {

    var path = require('path');
    var _ = require('underscore');
    var fs = require('fs');

    function is_in_array(arr,search){
        for( var n in arr ){
            if( arr[n] == search ) return true;
        }
        return false;
    }

    var meta_entry = (function(){
        return function(){
            this._base_dir = ''
            this.reference_time = ''
            this.build = []
            this.dependences = []
            this.extras = {}

            var that = this
            this.save = function(out_path, cb){
                that.update_dependences_addressing()
                that.dedup_dependencies()
                that.update_reference_time()
                var o = {}
                _.extend(o, that)
                delete o._base_dir
                var contents = JSON.stringify(o, null, 4)
                fs.mkdirSync(path.dirname(out_path))
                fs.writeFileSync(out_path, contents)
                if( cb ) cb(null)
            }
            this.require_task = function(build_cmd, options){
                var b = {}
                b[build_cmd] = options
                that.build.push(b)
                return that
            }
            this.is_fresh = function(){
                var time = that.reference_time
                var is_fresh = true
                for( var n in that.dependences ){
                    var filename = that.dependences[n]
                    if ( fs.existsSync(filename) ){
                        var ft = fs.statSync(filename);
                        is_fresh = is_fresh && ft.mtime <= time
                    }else{
                        is_fresh = false;
                    }
                }
{'fraction_non_alphanumeric': 0.09688581314878893, 'fraction_numerical': 0.0005767012687427913, 'mean_word_length': 1.1506509609423434, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241715', 'n_tokens_mistral': 939, 'n_tokens_neox': 861, 'n_words': 196}