<commit_before>from mediawords.test.test_database import TestDatabaseTestCase
from mediawords.tm.mine import postgres_regex_match


class TestTMMine(TestDatabaseTestCase):
    def test_postgres_regex_match(self):
        regex = '(?: [[:<:]]alt-right | [[:<:]]alt[[:space:]]+right | [[:<:]]alternative[[:space:]]+right )'

        # Match
        strings = ['This is a string describing alt-right and something else.']
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is True

        # No match
        strings = ['This is a string describing just something else.']
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is False

        # One matching string
        strings = [
            'This is a string describing something else.',
            'This is a string describing alt-right.',
        ]
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is True

        # Two non-matching strings
        strings = [
            'This is a string describing something else.',
            'This is a string describing something else again.',
        ]
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is False<commit_msg>Test changes made in 1a18c8c4c<commit_after>from mediawords.test.test_database import TestDatabaseTestCase
from mediawords.tm.mine import postgres_regex_match


class TestTMMine(TestDatabaseTestCase):
    def test_postgres_regex_match(self):
        regex = '(?: [[:<:]]alt-right | [[:<:]]alt[[:space:]]+right | [[:<:]]alternative[[:space:]]+right )'

        # Match
        strings = ['This is a string describing alt-right and something else.']
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is True

        # No match
        strings = ['This is a string describing just something else.']
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is False

        # One matching string
        strings = [
            'This is a string describing something else.',
            'This is a string describing alt-right.',
        ]
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is True

        # Two non-matching strings
        strings = [
            'This is a string describing something else.',
            'This is a string describing something else again.',
        ]
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is False

        # String over 1 MB
        strings = [
            'alt-right alt-right ' * (1024 * 1024)
        ]
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is True

        # String where regex is not in the first 1 MB
        strings = [
            'a' * (1024 * 1024 + 10) + ' alt-right alt-right'
        ]
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is False

        # String with a null char
        strings = [
            "alt-right alt-right \x00 alt-right"
        ]
        assert postgres_regex_match(db=self.db(), strings=strings, regex=regex) is False
{'fraction_non_alphanumeric': 0.110507824976046, 'fraction_numerical': 0.008623442989460236, 'mean_word_length': 2.706508875739645, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '806048', 'n_tokens_mistral': 890, 'n_tokens_neox': 853, 'n_words': 253}