<commit_before>#!/usr/bin/env python
"""
Convert tsv to json from stdin, assuming first line is header.

e.g., cat my.tsv | tsv2json
"""
import json
import sys

from . import open_read_utf8, open_write_utf8


def main():
    output = open_write_utf8(sys.stdout)
    header = None
    for line in open_read_utf8(sys.stdin):
        try:
            columns = line.strip().split('\t')
            if not header:
                header = columns
            else:
                d = {}
                for field, value in zip(header, columns):
                    if '.' in field:
                        parts = field.split('.')
                        r = d
                        for p in parts[:-1]:
                            if p not in r:
                                r[p] = {}
                            r = r[p]
                        r[parts[-1]] = value
                    else:
                        d[field] = value
                output.write('%s\n' % json.dumps(d, encoding='utf-8',
                                                 ensure_ascii=False))
        except Exception as e:
            sys.stderr.write('bad line %s\n' % e)
            print(sys.exc_info()[0])<commit_msg>Fixes python3 issue in json2tsv<commit_after>#!/usr/bin/env python
"""
Convert tsv to json from stdin, assuming first line is header.

e.g., cat my.tsv | tsv2json
"""
import json
import sys

from . import open_read_utf8, open_write_utf8


def main():
    output = open_write_utf8(sys.stdout)
    header = None
    for line in open_read_utf8(sys.stdin):
        try:
            columns = line.strip().split('\t')
            if not header:
                header = columns
            else:
                d = {}
                for field, value in zip(header, columns):
                    if '.' in field:
                        parts = field.split('.')
                        r = d
                        for p in parts[:-1]:
                            if p not in r:
                                r[p] = {}
                            r = r[p]
                        r[parts[-1]] = value
                    else:
                        d[field] = value
                output.write('%s\n' % json.dumps(d, ensure_ascii=False))
        except Exception as e:
            sys.stderr.write('bad line %s\n' % e)
            print(sys.exc_info()[0])
{'fraction_non_alphanumeric': 0.10622090562843843, 'fraction_numerical': 0.008040626322471435, 'mean_word_length': 1.0994671403197158, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2056332', 'n_tokens_mistral': 667, 'n_tokens_neox': 640, 'n_words': 178}