<commit_before>from __future__ import unicode_literals, print_function
from mf2py import Parser
from nose.tools import assert_equal, assert_true, assert_false
import os.path
import sys
import glob
import json
from unittest import TestCase
TestCase.maxDiff = None


if sys.version < '3':
    text_type = unicode
    binary_type = str
else:
    text_type = str
    binary_type = bytes


def parse_fixture(path, url=None):
    with open(os.path.join("test/examples/", path)) as f:
        p = Parser(doc=f, url=url)
        return p.to_dict()


def test_empty():
    p = Parser()
    assert_true(type(p) is not None)
    assert_true(type(p.to_dict()) is dict)


def test_open_file():
    p = Parser(doc=open("test/examples/empty.html"))
    assert_true(p.__doc__ is not None)
    assert_true(type(p) is not None)
    assert_true(type(p.to_dict()) is dict)


def test_user_agent():
    assert_equal(Parser.useragent, 'mf2py - microformats2 parser for python')
    Parser.useragent = 'something else'
    assert_equal(Parser.useragent, 'something else')
    # set back to default. damn stateful classes
    Parser.useragent = 'mf2py - microformats2 parser for python'


def test_base():
    p = Parser(doc=open("test/examples/base.html"))
    assert_equal(p.__url__, "http://tantek.com/")


def test_simple_parse():
    result = parse_fixture("simple_person_reference.html")
    assert_equal(result["items"][0]["properties"],
                 {'name': ['<NAME>']})


def test_simple_person_reference_implied():
    p = Parser(doc=open("test/examples/simple_person_reference_implied.html"))
    result = p.to_dict()
    assert_equal(result["items"][0]["properties"],
                 {'name': ['<NAME>']})


def test_simple_person_reference_same_element():
    result = parse_fixture("simple_person_reference_same_element.html")
    assert_equal(result["items"][0]["properties"],
                 {'name': ['<NAME>']})


def test_person_with_url():
    p = Parser(doc=open("test/examples/person_with_url.html"))
    result = p.to_dict()
    assert_equal(result["items"][0]["properties"]["name"],
                 ['<NAME>'])
    assert_equal(result["items"][0]["properties"]["url"],
                 ['http://tommorris.org/'])


def test_vcp():
    result = parse_fixture("value_class_person.html")
    assert_equal(result["items"][0]["properties"]["tel"], ['+44 1234 567890'])


def test_multiple_root_classnames():
    result = parse_fixture("nested_multiple_classnames.html")
    # order does not matter
    assert_equal(len(result["items"]), 1)
    assert_equal(set(result["items"][0]["type"]),
                 set(["h-entry", "h-as-note"]))


def test_property_nested_microformat():
    result = parse_fixture("nested_multiple_classnames.html")

    assert_equal(len(result["items"]), 1)
    assert "author" in result["items"][0]["properties"]
    assert_equal(
        result["items"][0]["properties"]["author"][0]["properties"]["name"][0],
        "<NAME>")
    assert_equal(
        result["items"][0]["properties"]["reviewer"][0]
        ["properties"]["name"][0],
        "<NAME>")
    assert_equal(
        result["items"][0]["properties"]["author"][0]
        ["properties"]["adr"][0]["properties"]["city"][0],
        "London")


def test_plain_child_microformat():
    result = parse_fixture("nested_multiple_classnames.html")

    assert_equal(len(result["items"]), 1)
    assert_true("children" in result["items"][0])
    assert_equal(len(result["items"][0]["children"]), 1)
    assert_equal(
        result["items"][0]["children"][0]["properties"]["name"][0],
        "Some Citation")


def test_implied_name():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][0]["properties"]["name"][0], "<NAME>")


def test_implied_url():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][1]["properties"]["url"][0],
                 "http://tommorris.org/")


def test_implied_nested_photo():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][2]["properties"]["photo"][0],
                 "http://tommorris.org/photo.png")


def test_implied_nested_photo_alt_name():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][3]["properties"]["name"][0], "<NAME>")


def test_implied_image():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][4]["properties"]["photo"][0],
                 "http://tommorris.org/photo.png")
    assert_equal(result["items"][4]["properties"]["name"][0], "<NAME>")


def test_datetime_parsing():
    result = parse_fixture("datetimes.html")
    assert_equal(result["items"][0]["properties"]["start"][0],
                 "2014-01-01T12:00:00+00:00")
    assert_equal(result["items"][0]["properties"]["end"][0],
                 "3014-01-01T18:00:00+00:00")
    assert_equal(result["items"][0]["properties"]["duration"][0],
                 "P1000Y")
    assert_equal(result["items"][0]["properties"]["updated"][0],
                 "2011-08-26T00:01:21+00:00")
    assert_equal(result["items"][0]["properties"]["updated"][1],
                 "2011-08-26T00:01:21+00:00")


def test_datetime_vcp_parsing():
    result = parse_fixture("datetimes.html")
    assert_equal(result["items"][1]["properties"]["published"][0],
                 "3014-01-01T01:21:00Z")
    assert_equal(result["items"][2]["properties"]["updated"][0],
                 "2014-03-11T09:55:00")
    assert_equal(result["items"][3]["properties"]["published"][0],
                 "2014-01-30T15:28:00")
    assert_equal(result["items"][4]["properties"]["published"][0],
                 "9999-01-14T11:52:00+08:00")<commit_msg>fix failing unit test resulting from datetime normalization change<commit_after>from __future__ import unicode_literals, print_function
from mf2py import Parser
from nose.tools import assert_equal, assert_true, assert_false
import os.path
import sys
import glob
import json
from unittest import TestCase
TestCase.maxDiff = None


if sys.version < '3':
    text_type = unicode
    binary_type = str
else:
    text_type = str
    binary_type = bytes


def parse_fixture(path, url=None):
    with open(os.path.join("test/examples/", path)) as f:
        p = Parser(doc=f, url=url)
        return p.to_dict()


def test_empty():
    p = Parser()
    assert_true(type(p) is not None)
    assert_true(type(p.to_dict()) is dict)


def test_open_file():
    p = Parser(doc=open("test/examples/empty.html"))
    assert_true(p.__doc__ is not None)
    assert_true(type(p) is not None)
    assert_true(type(p.to_dict()) is dict)


def test_user_agent():
    assert_equal(Parser.useragent, 'mf2py - microformats2 parser for python')
    Parser.useragent = 'something else'
    assert_equal(Parser.useragent, 'something else')
    # set back to default. damn stateful classes
    Parser.useragent = 'mf2py - microformats2 parser for python'


def test_base():
    p = Parser(doc=open("test/examples/base.html"))
    assert_equal(p.__url__, "http://tantek.com/")


def test_simple_parse():
    result = parse_fixture("simple_person_reference.html")
    assert_equal(result["items"][0]["properties"],
                 {'name': ['<NAME>']})


def test_simple_person_reference_implied():
    p = Parser(doc=open("test/examples/simple_person_reference_implied.html"))
    result = p.to_dict()
    assert_equal(result["items"][0]["properties"],
                 {'name': ['<NAME>']})


def test_simple_person_reference_same_element():
    result = parse_fixture("simple_person_reference_same_element.html")
    assert_equal(result["items"][0]["properties"],
                 {'name': ['<NAME>']})


def test_person_with_url():
    p = Parser(doc=open("test/examples/person_with_url.html"))
    result = p.to_dict()
    assert_equal(result["items"][0]["properties"]["name"],
                 ['<NAME>'])
    assert_equal(result["items"][0]["properties"]["url"],
                 ['http://tommorris.org/'])


def test_vcp():
    result = parse_fixture("value_class_person.html")
    assert_equal(result["items"][0]["properties"]["tel"], ['+44 1234 567890'])


def test_multiple_root_classnames():
    result = parse_fixture("nested_multiple_classnames.html")
    # order does not matter
    assert_equal(len(result["items"]), 1)
    assert_equal(set(result["items"][0]["type"]),
                 set(["h-entry", "h-as-note"]))


def test_property_nested_microformat():
    result = parse_fixture("nested_multiple_classnames.html")

    assert_equal(len(result["items"]), 1)
    assert "author" in result["items"][0]["properties"]
    assert_equal(
        result["items"][0]["properties"]["author"][0]["properties"]["name"][0],
        "<NAME>")
    assert_equal(
        result["items"][0]["properties"]["reviewer"][0]
        ["properties"]["name"][0],
        "<NAME>")
    assert_equal(
        result["items"][0]["properties"]["author"][0]
        ["properties"]["adr"][0]["properties"]["city"][0],
        "London")


def test_plain_child_microformat():
    result = parse_fixture("nested_multiple_classnames.html")

    assert_equal(len(result["items"]), 1)
    assert_true("children" in result["items"][0])
    assert_equal(len(result["items"][0]["children"]), 1)
    assert_equal(
        result["items"][0]["children"][0]["properties"]["name"][0],
        "Some Citation")


def test_implied_name():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][0]["properties"]["name"][0], "<NAME>")


def test_implied_url():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][1]["properties"]["url"][0],
                 "http://tommorris.org/")


def test_implied_nested_photo():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][2]["properties"]["photo"][0],
                 "http://tommorris.org/photo.png")


def test_implied_nested_photo_alt_name():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][3]["properties"]["name"][0], "<NAME>")


def test_implied_image():
    result = parse_fixture("implied_properties.html")
    assert_equal(result["items"][4]["properties"]["photo"][0],
                 "http://tommorris.org/photo.png")
    assert_equal(result["items"][4]["properties"]["name"][0], "<NAME>ris")


def test_datetime_parsing():
    result = parse_fixture("datetimes.html")
    assert_equal(result["items"][0]["properties"]["start"][0],
                 "2014-01-01T12:00:00+00:00")
    assert_equal(result["items"][0]["properties"]["end"][0],
                 "3014-01-01T18:00:00+00:00")
    assert_equal(result["items"][0]["properties"]["duration"][0],
                 "P1000Y")
    assert_equal(result["items"][0]["properties"]["updated"][0],
                 "2011-08-26T00:01:21+00:00")
    assert_equal(result["items"][0]["properties"]["updated"][1],
                 "2011-08-26T00:01:21+00:00")


def test_datetime_vcp_parsing():
    result = parse_fixture("datetimes.html")
    assert_equal(result["items"][1]["properties"]["published"][0],
                 "3014-01-01T01:21:00Z")
    assert_equal(result["items"][2]["properties"]["updated"][0],
                 "2014-03-11 09:55:00")
    assert_equal(result["items"][3]["properties"]["published"][0],
                 "2014-01-30T15:28:00")
    assert_equal(result["items"][4]["properties"]["published"][0],
                 "9999-01-14T11:52:00+08:00")
{'fraction_non_alphanumeric': 0.18227803942089657, 'fraction_numerical': 0.03680446537589395, 'mean_word_length': 4.174638989169675, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 2, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3906556', 'n_tokens_mistral': 3996, 'n_tokens_neox': 3862, 'n_words': 530}