<commit_before>#!/usr/bin/env python
"""
common tools for parsers
"""

import sys
import simplejson

def netstring(x):
    """
        >>> netstring('banana')
        '6:banana,'
    """
    return str(len(x)) + ':' + x + ','

def jsonify(d):
    return simplejson.dumps(d, indent=2, sort_keys=True)

def export(generator):
    for item in generator:
        sys.stdout.write(netstring(jsonify(item)))

if __name__ == "__main__":
    import doctest
    doctest.testmod()<commit_msg>add unexport function for njs format<commit_after>#!/usr/bin/env python
"""
common tools for parsers
"""

import sys
import simplejson

def netstring(x):
    """
        >>> netstring('banana')
        '6:banana,'
    """
    return str(len(x)) + ':' + x + ','

def jsonify(d):
    return simplejson.dumps(d, indent=2, sort_keys=True)

def export(generator):
    for item in generator:
        sys.stdout.write(netstring(jsonify(item)))

def unexport(fh):
    n = '0'
    while n:
        n = int(n + ''.join(c for c in iter(lambda: fh.read(1), ':')))
        yield simplejson.loads(fh.read(n))
        assert fh.read(1) == ','
        n = fh.read(1)

if __name__ == "__main__":
    import doctest
    doctest.testmod()
{'fraction_non_alphanumeric': 0.16486261448792672, 'fraction_numerical': 0.006661115736885929, 'mean_word_length': 3.006666666666667, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3703206', 'n_tokens_mistral': 438, 'n_tokens_neox': 430, 'n_words': 96}