<commit_before>import unittest
from pyramid import testing

class Test_lib_libapex(unittest.TestCase):
    def setUp(self):
        self.config = testing.setUp()

    def tearDown(self):
        testing.tearDown()

    #    request = testing.DummyRequest()
    #    request.context = testing.DummyResource()

    def test_apexid_from_url_google(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$G$xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', apexid_from_url('Google','https://www.google.com/accounts/o8/id?id=xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'))

    def test_apexid_from_url_twitter(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$T$xxxxxxxx', apexid_from_url('Twitter',"http://twitter.com/?id=['xxxxxxxx']"))

    def test_apexid_from_url_facebook(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$F$xxxxxxxxxx', apexid_from_url('Facebook','https://graph.facebook.com/xxxxxxxxxx'))

    def test_apexid_from_url_yahoo(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$Y$xxxxxxxxxxxxxxxxxx.xxxxx#xxxxx', apexid_from_url('Yahoo','https://me.yahoo.com/a/xxxxxxxxxxxxxxxxxx.xxxxx#xxxxx'))

    def test_apexid_from_url_invalid_provider(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual(None, apexid_from_url('Blah','https://www.google.com/accounts/o8/id?id=xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'))

    def test_apexid_from_url_invalid_url_google(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual(None, apexid_from_url('Google','https://www.google.com/accounts/o8/id?xx=xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'))

    def test_apexid_from_url_invalid_url_twitter(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual(None, apexid_from_url('Twitter',"http://twitter.com/?xx=['xxxxxxxx']"))

    def test_apexid_from_url_invalid_url_facebook(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual(None, apexid_from_url('Facebook','https://graph.facebook.com'))

    def test_apexid_from_url_invalid_url_yahoo(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual(None, apexid_from_url('Yahoo','https://me.yahoo.com/a'))
<commit_msg>consolidated tests for lib/libapex.py, general cleanup<commit_after>import unittest

from pyramid import testing

class Test_lib_libapex(unittest.TestCase):
    def setUp(self):
        self.config = testing.setUp()

    def tearDown(self):
        testing.tearDown()

    def test_apexid_from_url_google(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$G$xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', apexid_from_url('Google','https://www.google.com/accounts/o8/id?id=xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'))
        self.assertEqual(None, apexid_from_url('Google','https://www.google.com/accounts/o8/id?xx=xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'))

    def test_apexid_from_url_twitter(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$T$xxxxxxxx', apexid_from_url('Twitter',"http://twitter.com/?id=['xxxxxxxx']"))
        self.assertEqual(None, apexid_from_url('Twitter',"http://twitter.com/?xx=['xxxxxxxx']"))

    def test_apexid_from_url_facebook(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$F$xxxxxxxxxx', apexid_from_url('Facebook','https://graph.facebook.com/xxxxxxxxxx'))
        self.assertEqual(None, apexid_from_url('Facebook','https://graph.facebook.com'))

    def test_apexid_from_url_yahoo(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual('$Y$xxxxxxxxxxxxxxxxxx.xxxxx#xxxxx', apexid_from_url('Yahoo','https://me.yahoo.com/a/xxxxxxxxxxxxxxxxxx.xxxxx#xxxxx'))
        self.assertEqual(None, apexid_from_url('Yahoo','https://me.yahoo.com/a'))

    def test_apexid_from_url_invalid_provider(self):
        from apex.lib.libapex import apexid_from_url

        self.assertEqual(None, apexid_from_url('Blah','https://www.google.com/accounts/o8/id?id=xxxxxxx-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx'))
{'fraction_non_alphanumeric': 0.12437692855447424, 'fraction_numerical': 0.0014241633040588653, 'mean_word_length': 4.679245283018868, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 14, 'lorem ipsum': 0, 'www.': 6, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 30, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4528558', 'n_tokens_mistral': 1641, 'n_tokens_neox': 1351, 'n_words': 159}