<commit_before>import os.path
import unittest

from adjunct import discovery, ogp

HERE = os.path.dirname(__file__)


class OPGTest(unittest.TestCase):
    def setUp(self):
        with open(os.path.join(HERE, "ogp.html"), "rb") as fh:
            self.properties = discovery.Extractor.extract(fh).properties
        self.root = ogp.Root()
        for prop, content in self.properties:
            self.root.insert(prop, content)

    def test_size(self):
        self.assertEqual(len(self.properties), 16)
        self.assertEqual(len(self.properties), len(list(self.root.flatten())))

    def test_access(self):
        self.assertSetEqual(set(self.root.attrs.keys()), {"og", "twitter"})
        self.assertEqual(str(self.root.get("og:type")), "song")

    def test_invalid(self):
        elem = self.root.get("og:audio")
        self.assertEqual(len(elem), 1)
        # Ensure this doesn't add anything
        self.assertEqual(len(self.properties), len(list(self.root.flatten())))
        self.assertIsInstance(elem, ogp.SingleValue)
        self.assertIsNone(elem.content)
        self.assertDictEqual(elem.attrs, {})
<commit_msg>Use existing OGP population code<commit_after>import os.path
import unittest

from adjunct import discovery, ogp

HERE = os.path.dirname(__file__)


class OPGTest(unittest.TestCase):
    def setUp(self):
        with open(os.path.join(HERE, "ogp.html"), "rb") as fh:
            self.properties = discovery.Extractor.extract(fh).properties
        self.root = ogp.Root.from_list(self.properties)

    def test_size(self):
        self.assertEqual(len(self.properties), 16)
        self.assertEqual(len(self.properties), len(list(self.root.flatten())))

    def test_access(self):
        self.assertSetEqual(set(self.root.attrs.keys()), {"og", "twitter"})
        self.assertEqual(str(self.root.get("og:type")), "song")

    def test_invalid(self):
        elem = self.root.get("og:audio")
        self.assertEqual(len(elem), 1)
        # Ensure this doesn't add anything
        self.assertEqual(len(self.properties), len(list(self.root.flatten())))
        self.assertIsInstance(elem, ogp.SingleValue)
        self.assertIsNone(elem.content)
        self.assertDictEqual(elem.attrs, {})

{'fraction_non_alphanumeric': 0.14298561151079137, 'fraction_numerical': 0.002697841726618705, 'mean_word_length': 3.320388349514563, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2050122', 'n_tokens_mistral': 705, 'n_tokens_neox': 688, 'n_words': 121}