<commit_before>from __future__ import unicode_literals

from conllu.models import TokenList
from conllu.parser import parse_token_and_metadata


def parse(data, fields=None):
    return [
        TokenList(*parse_token_and_metadata(sentence, fields=fields))
        for sentence in data.split("\n\n")
        if sentence
    ]


def _iter_sents(in_file):
    buf = []
    for line in in_file:
        if line == "\n":
            yield "".join(buf)[:-1]
            buf = []
        else:
            buf.append(line)
    if buf:
        yield "".join(buf)


def parse_incr(in_file, fields=None):
    for sentence in _iter_sents(in_file):
        yield TokenList(*parse_token_and_metadata(sentence, fields=fields))


def parse_tree(data):
    tokenlists = parse(data)

    sentences = []
    for tokenlist in tokenlists:
        sentences.append(tokenlist.to_tree())

    return sentences


def parse_tree_incr(in_file):
    for tokenlist in parse_incr(in_file):
        yield tokenlist.to_tree()<commit_msg>Move helper to end of module and remove double spaces.<commit_after>from __future__ import unicode_literals

from conllu.models import TokenList
from conllu.parser import parse_token_and_metadata


def parse(data, fields=None):
    return [
        TokenList(*parse_token_and_metadata(sentence, fields=fields))
        for sentence in data.split("\n\n")
        if sentence
    ]

def parse_incr(in_file, fields=None):
    for sentence in _iter_sents(in_file):
        yield TokenList(*parse_token_and_metadata(sentence, fields=fields))

def parse_tree(data):
    tokenlists = parse(data)

    sentences = []
    for tokenlist in tokenlists:
        sentences.append(tokenlist.to_tree())

    return sentences

def parse_tree_incr(in_file):
    for tokenlist in parse_incr(in_file):
        yield tokenlist.to_tree()

def _iter_sents(in_file):
    buf = []
    for line in in_file:
        if line == "\n":
            yield "".join(buf)[:-1]
            buf = []
        else:
            buf.append(line)
    if buf:
        yield "".join(buf)
{'fraction_non_alphanumeric': 0.09887968826108134, 'fraction_numerical': 0.000974184120798831, 'mean_word_length': 2.8975332068311195, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6466570', 'n_tokens_mistral': 656, 'n_tokens_neox': 641, 'n_words': 154}