<commit_before>// The complete reformatter in javascript
var sys = require('sys');
var posix = require('posix');
var yajl = require('./src/yajl');

var callbacks = {
    onNull : function(gen) {
        gen.addNull();
    },
    onBoolean : function(gen, bool) {
        gen.boolean( bool );
    },
    onNumber : function(gen, str) {
        gen.number( str );
    },
    onString : function(gen, str) {
        gen.string( str );
    },
    onMapKey : function(gen, key) {
        gen.string( key );
    },
    onStartMap : function(gen) {
        g.openMap();
    },
    onEndMap : function(gen) {
        g.closeMap();
    },
    onStartArray : function(gen) {
        g.openArray();
    },
    onEndArray : function(gen) {
        g.closeArray();
    }
};

function usage() {
    sys.error( "Usage: %s <filename>\n"
               +"    -m Minimize json rather than beautify (default)\n"
               +"    -u allow invalid UTF8 inside strings during parsing\n");
    process.exit(1);
}

var conf = {
    beautify: 1,
    indentString: "  "
};

var cfg = {
    allowComments: 1,
    checkUTF8: 1
};

if( process.ARGV.length == 3 ) {
    if( process.ARGV[2] == "-m" )
        conf.beautify = 0;
    else if( process.ARGV[2] == "-u" )
        cfg.checkUTF8 = 0;
    else
        usage();
}
else if( process.ARGV.length != 2 ) {
    usage();
}

var g = new yajl.Generator( conf, null );

var handle = yajl.createHandle();
handle.addListener( "null", function() {
    sys.debug("Nullify");
});

var done = false;

process.stdio.open();
process.stdio.addListener( "data", function( data ) {
    var stat = handle.parse(data);
//    if( stat != yajl.STATUS_OK &&
//        stat != yajl.STATUS_INSUFFICIENT_DATA ) {
//        sys.error( handle.error( yajl.ERROR_VERBOSE, data ) );
//    }
});
//
//process.stdio.addListener( "close", function() {
//    var stat = handle.parseComplete();
//    if( stat != yajl.STATUS_OK )
//        sys.error( handle.error( yajl.ERROR_VERBOSE, data ) );
//    else
//        process.stdio.write( g.buffer() );
//});<commit_msg>made test.js more javascripty by adding events, removed error checking<commit_after>// Will be a complete reformatter in javascript similar to the Yajl reformatter
var sys = require('sys');
var posix = require('posix');
var yajl = require('./src/yajl');

function usage() {
    sys.error( "Usage: %s <filename>\n"
               +"    -m Minimize json rather than beautify (default)\n"
               +"    -u allow invalid UTF8 inside strings during parsing\n");
    process.exit(1);
}

var conf = {
    beautify: 1,
    indentString: "  "
};

var cfg = {
    allowComments: 1,
    checkUTF8: 1
};

if( process.ARGV.length == 3 ) {
    if( process.ARGV[2] == "-m" )
        conf.beautify = 0;
    else if( process.ARGV[2] == "-u" )
        cfg.checkUTF8 = 0;
    else
        usage();
}
else if( process.ARGV.length != 2 ) {
    usage();
}

var handle = yajl.createHandle();
handle.addListener( "null", function() {
    sys.debug("Nullify");
});

handle.addListener( "boolean", function(b) {
    sys.debug(b);
});

handle.addListener( "number", function(n) {
    sys.debug("Num: " + n);
});

handle.addListener( "string", function(s) {
    sys.debug("String: " + s);
});

handle.addListener( "mapKey", function(k) {
    sys.debug("Key: " + k );
});

handle.addListener( "startMap", function() {
    sys.debug("{");
});

handle.addListener( "endMap", function() {
    sys.debug("}");
});

handle.addListener( "startArray", function() {
    sys.debug("[");
});

handle.addListener( "endArray", function() {
    sys.debug("]");
});

handle.addListener( "error", function( errorString ) {
    sys.error("Error parsing json: " + errorString );
});

var done = false;

process.stdio.open();
process.stdio.addListener( "data", function( data ) {
    handle.parse(data);
});
process.stdio.addListener( "close", function() {
    handle.parseComplete();
});
{'fraction_non_alphanumeric': 0.1793297518546943, 'fraction_numerical': 0.00665131747249936, 'mean_word_length': 2.657623947614593, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6548391', 'n_tokens_mistral': 1367, 'n_tokens_neox': 1328, 'n_words': 303}