<commit_before># coding: utf-8
from __future__ import unicode_literals

import unittest
from collections import OrderedDict
from textwrap import dedent

from conllu.models import TokenList
from conllu.parser import (
    DEFAULT_FIELDS, ParseException, head_to_token, parse_comment_line, parse_dict_value, parse_id_value,
    parse_int_value, parse_line, parse_nullable_value, parse_paired_list_value, parse_token_and_metadata, serialize,
    serialize_field,
)


class TestParse(unittest.TestCase):
    def test_empty(self):
        with self.assertRaises(ParseException):
            parse_token_and_metadata(None)

    def test_newlines_in_sentence(self):
        data = dedent("""\
            # meta = data
            1\thej
            2\tdå

            3\thej
            4\tdå
        """)
        tokens, metadata = parse_token_and_metadata(data)
        self.assertListEqual(tokens, [
            OrderedDict([("id", 1), ("form", "hej")]),
            OrderedDict([("id", 2), ("form", "då")]),
            OrderedDict([("id", 3), ("form", "hej")]),
            OrderedDict([("id", 4), ("form", "då")]),
        ])
        self.assertEqual(metadata, OrderedDict([("meta", "data")]))

    def test_invalid_metadata(self):
        data = dedent("""\
            # meta = data2
            # meta = data
            # newdoc
            # newpar
            # meta
            # = data
        """)
        _, metadata = parse_token_and_metadata(data)
        self.assertEqual(metadata, OrderedDict([("meta", "data"), ("newdoc", None), ("newpar", None)]))

class TestParseLine(unittest.TestCase):
    def test_empty(self):
        with self.assertRaises(ParseException):
            line = "invalid_id\t_\t_\t_\t_\t_\t_\t_\t_\t"
            parse_line(line, fields=DEFAULT_FIELDS)

    def test_parse_line(self):
        line = "1\tThe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t4\tdet\t_\t_"
        self.assertEqual(
            parse_line(line, fields=DEFAULT_FIELDS),<commit_msg>Add more tests for custom fields.<commit_after># coding: utf-8
from __future__ import unicode_literals

import unittest
from collections import OrderedDict
from textwrap import dedent

from conllu.models import TokenList
from conllu.parser import (
    DEFAULT_FIELDS, ParseException, head_to_token, parse_comment_line, parse_dict_value, parse_id_value,
    parse_int_value, parse_line, parse_nullable_value, parse_paired_list_value, parse_token_and_metadata, serialize,
    serialize_field,
)


class TestParse(unittest.TestCase):
    def test_empty(self):
        with self.assertRaises(ParseException):
            parse_token_and_metadata(None)

    def test_newlines_in_sentence(self):
        data = dedent("""\
            # meta = data
            1\thej
            2\tdå

            3\thej
            4\tdå
        """)
        tokens, metadata = parse_token_and_metadata(data)
        self.assertListEqual(tokens, [
            OrderedDict([("id", 1), ("form", "hej")]),
            OrderedDict([("id", 2), ("form", "då")]),
            OrderedDict([("id", 3), ("form", "hej")]),
            OrderedDict([("id", 4), ("form", "då")]),
        ])
        self.assertEqual(metadata, OrderedDict([("meta", "data")]))

    def test_invalid_metadata(self):
        data = dedent("""\
            # meta = data2
            # meta = data
            # newdoc
            # newpar
            # meta
            # = data
        """)
        _, metadata = parse_token_and_metadata(data)
        self.assertEqual(metadata, OrderedDict([("meta", "data"), ("newdoc", None), ("newpar", None)]))

    def test_custom_fields(self):
        data = dedent("""\
            1\t1\t1
            2\t2\t2
        """)
        tokens, _ = parse_token_and_metadata(data, fields=("id", "id", "id"))
        self.assertEqual(tokens, [
            OrderedDict([("id", 1), ("id", 1), ("id", 1)]),
            OrderedDict([("id", 2), ("id", 2), ("id", 2)]),
        ])

class TestParseLine(unittest.TestCase):
    def test_empty(self):
        with self.assertRaises(ParseException):
            line = "invalid_id\t_\t_\t_\t_\t_\t_\t_\t_\t"
            parse_line(line, fields=DEFAULT_FIELDS)

    def test_parse_line(self):
        line = "1\tThe\tthe\tDET\tDT\tDefinite=Def|PronType=Art\t4\tdet\t_\t_"
        self.assertEqual(
            parse_line(line, fields=DEFAULT_FIELDS),
{'fraction_non_alphanumeric': 0.13718742830924524, 'fraction_numerical': 0.008258774948382657, 'mean_word_length': 2.504823151125402, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '471939', 'n_tokens_mistral': 1412, 'n_tokens_neox': 1374, 'n_words': 258}