<commit_before>from py2neo import Node

def check_equals(x, y):
    assert x == y

def check_exists(x, attribute):
    try:
        x[attribute]
    except KeyError:
        assert False, "missing key {}".format(attribute)

def check_not_exists(x, attribute):
    try:
        x[attribute]
    except KeyError:
        assert True, "key {} unexpectedly exists".format(attribute)

def check_ad_equals_expected(ad, expected, ignore_fields=[]):
    # check that all expected keys are present
    for key in expected:
        assert (ad.properties[key] is not None), ("{} is missing".format(key))

    for key in ad.properties:
        a = ad[key]
        check_exists(expected, key)
        b = expected[key]
        if key in ignore_fields:
            continue
        if isinstance(a, list):
            a.sort()
        if isinstance(b, list):
            b.sort()
        check_equals(a, b)

class Runner(object):
    @classmethod
    def setup_class(cls, EnhancerToTest, test_nodes):
        cls.ENHANCER = EnhancerToTest()
        # seed some data
        for node in test_nodes:
            cls.ENHANCER.db.create(Node("Ad", **node))

    @classmethod
    def teardown_class(cls):
        cls.ENHANCER.db.delete_all()

    def test_run(self, expected_results, ignore_fields=[]):
        """
        Setting a list of ignore_fields first checks that the fields
        exist before ignoring their values. This is useful for instance,
        when comparing "tweets" which may change over time.
        """
        self.ENHANCER.run(skip_on_error=False)

        # use the ENHANCER's DB to check that results are as expected
        records = self.ENHANCER.db.cypher.execute("MATCH (ad) RETURN ad")

        # first, check that we haven't *added* any records
        yield check_equals, len(records), len(expected_results)

        # second, check that the records contain the expected fields
        for result, exp in zip(records, expected_results):
            yield check_ad_equals_expected, result.ad, exp, ignore_fields<commit_msg>changes the way tests are run<commit_after>from py2neo import Node
import nose.tools

def check_exists(x, attribute):
    try:
        x[attribute]
    except KeyError:
        assert False, "missing key {}".format(attribute)

def check_not_exists(x, attribute):
    try:
        x[attribute]
    except KeyError:
        assert True, "key {} unexpectedly exists".format(attribute)

class Runner(object):
    # subclasses have to set these fields
    ENHANCER = None
    OUTPUT = []
    INPUT = []
    IGNORE = [] # this is optional, setting a list of IGNORE
                # first checks that the fields exist before ignoring
                # their values. This is useful for instance, when
                # comparing "tweets" which may change over time.

    def check_ad_equals_expected(self, test_node, expected, ignore_fields=[]):
        try:
            enhancer = self.ENHANCER()
            enhancer.db.create(Node("Ad", **test_node))

            enhancer.run(skip_on_error=False)

            # use the ENHANCER's DB to check that results are as expected
            records = enhancer.db.cypher.execute("MATCH (ad) RETURN ad")

            assert len(records)==1, "We're expecting only one entry in the DB"

            ad = records[0].ad

            # check that all expected keys are present
            for key in expected:
                assert (ad.properties[key] is not None), ("{} is missing".format(key))

            for key in ad.properties:
                a = ad[key]
                check_exists(expected, key)
                b = expected[key]
                if key in ignore_fields:
                    continue
                if isinstance(a, list):
                    a.sort()
                if isinstance(b, list):
                    b.sort()
                nose.tools.eq_(a, b)
        finally:
            enhancer.db.delete_all()

    def test_run(self):
        # Populates the DB one by one and runs the enhancer on it.
        # This structure lets us see captured log outputs.

        # check that the records contain the expected fields
        for test_node, exp in zip(self.INPUT, self.OUTPUT):
            yield self.check_ad_equals_expected, test_node, exp, self.IGNORE
{'fraction_non_alphanumeric': 0.08820079756040347, 'fraction_numerical': 0.0009383063570255688, 'mean_word_length': 2.1844660194174756, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4352059', 'n_tokens_mistral': 1225, 'n_tokens_neox': 1173, 'n_words': 387}