<commit_before>import stream from '../../lib/stream/index';
import jsonparser from '../../lib/standard/jsonparser';


/*
  ======== A Handy Little Nodeunit Reference ========
  https://github.com/caolan/nodeunit

  Test methods:
    test.expect(numAssertions)
    test.done()
  Test assertions:
    test.ok(value, [message])
    test.equal(actual, expected, [message])
    test.notEqual(actual, expected, [message])
    test.deepEqual(actual, expected, [message])
    test.notDeepEqual(actual, expected, [message])
    test.strictEqual(actual, expected, [message])
    test.notStrictEqual(actual, expected, [message])
    test.throws(block, [error], [message])
    test.doesNotThrow(block, [error], [message])
    test.ifError(value)
*/

export default {
  setUp: function(done) {
    done();
  },
    
  'number accepted': function(test) {
    test.expect(1);
    // tests here    
    test.ok(jsonparser.parse(stream.ofString('123')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'string accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('"123"')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'string and unrecognized item rejected': function(test) {
    test.expect(1);
    // tests here
    test.equal(jsonparser.parse(stream.ofString('"123" -')).isAccepted(), 
               false,
               'should be rejected.');
    test.done();
  },
    
  'string and unrecognized item rejected with correct offset': function(test) {
    test.expect(1);
    // tests here
    var result = jsonparser.parse(stream.ofString('["123", -]'));
    test.equal(result.offset,
               7,
               'should be 7.');
    test.done();
  },
    
  'null accepted': function(test) {
    test.expect(1);
    // tests here    
    test.ok(jsonparser.parse(stream.ofString('null')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
 
  'true accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('true')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'false accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('false')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'empty array accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('[ ]')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'singleton array accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('[ 123 ]')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'multi element array accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('[ 123 , 234 ]')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'empty object accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('{ }')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'singleton object accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('{ "a" : "v" }')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'multi element object accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('{ "a" : "v", "a" : [] }')).isAccepted(), 
            'should be accepted.');
    test.done();
  }

};<commit_msg>Adding a test for multi-level json<commit_after>import stream from '../../lib/stream/index';
import jsonparser from '../../lib/standard/jsonparser';


/*
  ======== A Handy Little Nodeunit Reference ========
  https://github.com/caolan/nodeunit

  Test methods:
    test.expect(numAssertions)
    test.done()
  Test assertions:
    test.ok(value, [message])
    test.equal(actual, expected, [message])
    test.notEqual(actual, expected, [message])
    test.deepEqual(actual, expected, [message])
    test.notDeepEqual(actual, expected, [message])
    test.strictEqual(actual, expected, [message])
    test.notStrictEqual(actual, expected, [message])
    test.throws(block, [error], [message])
    test.doesNotThrow(block, [error], [message])
    test.ifError(value)
*/

export default {
  setUp: function(done) {
    done();
  },
    
  'number accepted': function(test) {
    test.expect(1);
    // tests here    
    test.ok(jsonparser.parse(stream.ofString('123')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'string accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('"123"')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'string and unrecognized item rejected': function(test) {
    test.expect(1);
    // tests here
    test.equal(jsonparser.parse(stream.ofString('"123" -')).isAccepted(), 
               false,
               'should be rejected.');
    test.done();
  },
    
  'string and unrecognized item rejected with correct offset': function(test) {
    test.expect(1);
    // tests here
    var result = jsonparser.parse(stream.ofString('["123", -]'));
    test.equal(result.offset,
               7,
               'should be 7.');
    test.done();
  },
    
  'null accepted': function(test) {
    test.expect(1);
    // tests here    
    test.ok(jsonparser.parse(stream.ofString('null')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
 
  'true accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('true')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'false accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('false')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'empty array accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('[ ]')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'singleton array accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('[ 123 ]')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'multi element array accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('[ 123 , 234 ]')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'empty object accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('{ }')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'singleton object accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('{ "a" : "v" }')).isAccepted(), 
            'should be accepted.');
    test.done();
  },
    
  'multi element object accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('{ "a" : "v", "a" : [] }')).isAccepted(), 
            'should be accepted.');
    test.done();
  },

  'multi level object accepted': function(test) {
    test.expect(1);
    // tests here
    test.ok(jsonparser.parse(stream.ofString('{ "a" : "v", "b" : {"c":{"d":12} }}')).isAccepted(),
        'should be accepted.');
    test.done();
  },

};
{'fraction_non_alphanumeric': 0.17835051546391753, 'fraction_numerical': 0.009664948453608248, 'mean_word_length': 2.5930555555555554, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5199897', 'n_tokens_mistral': 2481, 'n_tokens_neox': 2488, 'n_words': 487}