<commit_before>import numpy
import codecs
import json

from libc.string cimport memset


def read_json_file(loc):
    paragraphs = []
    for doc in json.load(open(loc)):
        for paragraph in doc['paragraphs']:
            words = []
            ids = []
            tags = []
            heads = []
            labels = []
            iob_ents = []
            for token in paragraph['tokens']:
                words.append(token['orth'])
                ids.append(token['start'])
                tags.append(token['tag'])
                heads.append(token['head'] if token['head'] >= 1 else token['start'])
                labels.append(token['dep'])
                iob_ents.append(token.get('iob_ent', 'O'))

            brackets = []
            tokenized = [s.replace('<SEP>', ' ').split(' ')
                         for s in paragraph['segmented'].split('<SENT>')]
            paragraphs.append((paragraph['raw'],
                tokenized,
                (ids, words, tags, heads, labels, _iob_to_biluo(iob_ents)),
                brackets))
    return paragraphs


def read_conll03_file(loc):
    sents = []
    text = codecs.open(loc, 'r', 'utf8').read().strip()
    for doc in text.split('-DOCSTART- -X- O O'):
        doc = doc.strip()
        if not doc:
            continue
        for sent_str in doc.split('\n\n'):
            words = []
            tags = []
            iob_ents = []
            ids = []
            lines = sent_str.strip().split('\n')
            idx = 0
            for line in lines:<commit_msg>* Fix off-by-one in head reading<commit_after>import numpy
import codecs
import json

from libc.string cimport memset


def read_json_file(loc):
    paragraphs = []
    for doc in json.load(open(loc)):
        for paragraph in doc['paragraphs']:
            words = []
            ids = []
            tags = []
            heads = []
            labels = []
            iob_ents = []
            for token in paragraph['tokens']:
                #print token['start'], token['orth'], token['head'], token['dep']
                words.append(token['orth'])
                ids.append(token['start'])
                tags.append(token['tag'])
                heads.append(token['head'] if token['head'] >= 0 else token['start'])
                labels.append(token['dep'])
                iob_ents.append(token.get('iob_ent', 'O'))

            brackets = []
            tokenized = [s.replace('<SEP>', ' ').split(' ')
                         for s in paragraph['segmented'].split('<SENT>')]
            paragraphs.append((paragraph['raw'],
                tokenized,
                (ids, words, tags, heads, labels, _iob_to_biluo(iob_ents)),
                brackets))
    return paragraphs


def read_conll03_file(loc):
    sents = []
    text = codecs.open(loc, 'r', 'utf8').read().strip()
    for doc in text.split('-DOCSTART- -X- O O'):
        doc = doc.strip()
        if not doc:
            continue
        for sent_str in doc.split('\n\n'):
            words = []
            tags = []
            iob_ents = []
            ids = []
            lines = sent_str.strip().split('\n')
            idx = 0
            for line in lines:
{'fraction_non_alphanumeric': 0.14653278945716974, 'fraction_numerical': 0.0031377470975839346, 'mean_word_length': 1.5688960515713135, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6474596', 'n_tokens_mistral': 931, 'n_tokens_neox': 893, 'n_words': 205}