<commit_before>import datetime
import os
from dataclasses import dataclass
from pathlib import Path
from unittest import TestCase, mock
from urllib.parse import urlparse, parse_qs

from autobot.autobot import englishify_time, PostAnalyzer
from autobot.config import Settings
from autobot.util.reddit_util import SubredditTool


@dataclass
class FakeSubmission:
    title: str = "A reddit title"
    selftext: str = "A reddit text"
    link_flair_text: str = ""


class TestBotMethods(TestCase):
    def _get_files_dir(self) -> Path:
        return Path(__file__).resolve().parent / "files"

    def test_reject_nsfw_in_title(self):
        """Test that the presence of "nsfw" in titles is a rejection"""
        analyzer = PostAnalyzer()
        self.assertTrue(analyzer.contains_nsfw_title("blah blah nsfw blah"))
        self.assertFalse(analyzer.contains_nsfw_title("NsFw_title_in_bars"))
        self.assertTrue(analyzer.contains_nsfw_title("nsfw leading title"))
        self.assertTrue(analyzer.contains_nsfw_title("Title ending with NSFW"))
        self.assertFalse(analyzer.contains_nsfw_title("No bad words"))
        self.assertTrue(analyzer.contains_nsfw_title("Has an [NSFW] tag"))
        self.assertTrue(analyzer.contains_nsfw_title("!NSFW!"))
        self.assertTrue(analyzer.contains_nsfw_title("Is this post NSFW?"))
        self.assertTrue(analyzer.contains_nsfw_title("Hi [Part 2] [NSFW]"))

    def test_reject_long_paragraphs(self):
        """This test asserts that paragraphs > (length) words are rejected."""
        analyzer = PostAnalyzer()
        # Basic test with just words
        text = " ".join(["text"] * 351)
        self.assertTrue(analyzer.contains_long_paragraphs([text]))

        # More advanced with punctuations and such.
        # In a naive implementation, word counting
        # could be affected by just splitting on spaces
        # and then things like standalone & would be counted.
        # i.e. len(text.split())
        add_text = " ".join(["text"] * 349)
        add_text += "& -- text"
        self.assertFalse(analyzer.contains_long_paragraphs([add_text]))

    def test_live_story_paragraphs(self):
        # Soul Cancer initiated issue #13
        files_dir = self._get_files_dir()
        analyzer = PostAnalyzer()
        with open(files_dir / "soul_cancer.md", "r") as sc:
            story = sc.read()
            submission = FakeSubmission(self_text=story)
            meta = analyzer.analyze(submission)
            self.assertFalse(meta.has_long_paragraphs)

    def test_chezecaek_full_story(self):
        # Chezecaek initiated issue #17
        files_dir = self._get_files_dir()
        analyzer = PostAnalyzer()
        with open(files_dir / "chezecaek.md", "r") as sc:
            story = sc.read()
            submission = FakeSubmission(self_text=story)
            meta = analyzer.analyze(submission)
            self.assertFalse(meta.has_codeblocks)

    def test_reject_long_paragraphs_funky_newlines(self):
        """Test edge case long paragraphs (newlines)"""

        # This test sees if paragraphs that have crappy line breaks
        # are accepted correctly.
        analyzer = PostAnalyzer()
        text = " ".join(["text"] * 300)
        text += "\n \n"
        text += " ".join(['more'] * 100)
        meta = analyzer.analyze(FakeSubmission(self_text=text))
        self.assertFalse(meta.is_invalid())

    def test_contains_codeblocks(self):
        """Test for codeblocks in a message"""

        code_opening = " " * 4
        analyzer = PostAnalyzer()
        text = f"{code_opening}This starts with four spaces"
        self.assertTrue(analyzer.contains_codeblocks([text]))

        tab_text = "\tThis starts with a tab"
        self.assertTrue(analyzer.contains_codeblocks([tab_text]))

        varied_spaces = "   \tThis has three spaces and a tab"
        self.assertTrue(analyzer.contains_codeblocks([varied_spaces]))
        self.assertFalse(analyzer.contains_codeblocks([" " * 8]))

        self.assertFalse(analyzer.contains_codeblocks([""]))

    def test_categorize_tags(self):
        title = "This is a sample post (volume 1) {part 2} |part 3|"
        analyzer = PostAnalyzer()
        series, final, invalid = analyzer.categorize_tags(title)
        self.assertEqual(len(invalid), 0, f"Unexpected bad tags: {invalid}")
        self.assertTrue(series)
        self.assertFalse(final)

    def test_mixed_series_final(self):<commit_msg>[easy] self_text -> selftext in tests<commit_after>import datetime
import os
from dataclasses import dataclass
from pathlib import Path
from unittest import TestCase, mock
from urllib.parse import urlparse, parse_qs

from autobot.autobot import englishify_time, PostAnalyzer
from autobot.config import Settings
from autobot.util.reddit_util import SubredditTool


@dataclass
class FakeSubmission:
    title: str = "A reddit title"
    selftext: str = "A reddit text"
    link_flair_text: str = ""


class TestBotMethods(TestCase):
    def _get_files_dir(self) -> Path:
        return Path(__file__).resolve().parent / "files"

    def test_reject_nsfw_in_title(self):
        """Test that the presence of "nsfw" in titles is a rejection"""
        analyzer = PostAnalyzer()
        self.assertTrue(analyzer.contains_nsfw_title("blah blah nsfw blah"))
        self.assertFalse(analyzer.contains_nsfw_title("NsFw_title_in_bars"))
        self.assertTrue(analyzer.contains_nsfw_title("nsfw leading title"))
        self.assertTrue(analyzer.contains_nsfw_title("Title ending with NSFW"))
        self.assertFalse(analyzer.contains_nsfw_title("No bad words"))
        self.assertTrue(analyzer.contains_nsfw_title("Has an [NSFW] tag"))
        self.assertTrue(analyzer.contains_nsfw_title("!NSFW!"))
        self.assertTrue(analyzer.contains_nsfw_title("Is this post NSFW?"))
        self.assertTrue(analyzer.contains_nsfw_title("Hi [Part 2] [NSFW]"))

    def test_reject_long_paragraphs(self):
        """This test asserts that paragraphs > (length) words are rejected."""
        analyzer = PostAnalyzer()
        # Basic test with just words
        text = " ".join(["text"] * 351)
        self.assertTrue(analyzer.contains_long_paragraphs([text]))

        # More advanced with punctuations and such.
        # In a naive implementation, word counting
        # could be affected by just splitting on spaces
        # and then things like standalone & would be counted.
        # i.e. len(text.split())
        add_text = " ".join(["text"] * 349)
        add_text += "& -- text"
        self.assertFalse(analyzer.contains_long_paragraphs([add_text]))

    def test_live_story_paragraphs(self):
        # Soul Cancer initiated issue #13
        files_dir = self._get_files_dir()
        analyzer = PostAnalyzer()
        with open(files_dir / "soul_cancer.md", "r") as sc:
            story = sc.read()
            submission = FakeSubmission(selftext=story)
            meta = analyzer.analyze(submission)
            self.assertFalse(meta.has_long_paragraphs)

    def test_chezecaek_full_story(self):
        # Chezecaek initiated issue #17
        files_dir = self._get_files_dir()
        analyzer = PostAnalyzer()
        with open(files_dir / "chezecaek.md", "r") as sc:
            story = sc.read()
            submission = FakeSubmission(selftext=story)
            meta = analyzer.analyze(submission)
            self.assertFalse(meta.has_codeblocks)

    def test_reject_long_paragraphs_funky_newlines(self):
        """Test edge case long paragraphs (newlines)"""

        # This test sees if paragraphs that have crappy line breaks
        # are accepted correctly.
        analyzer = PostAnalyzer()
        text = " ".join(["text"] * 300)
        text += "\n \n"
        text += " ".join(['more'] * 100)
        meta = analyzer.analyze(FakeSubmission(selftext=text))
        self.assertFalse(meta.is_invalid())

    def test_contains_codeblocks(self):
        """Test for codeblocks in a message"""

        code_opening = " " * 4
        analyzer = PostAnalyzer()
        text = f"{code_opening}This starts with four spaces"
        self.assertTrue(analyzer.contains_codeblocks([text]))

        tab_text = "\tThis starts with a tab"
        self.assertTrue(analyzer.contains_codeblocks([tab_text]))

        varied_spaces = "   \tThis has three spaces and a tab"
        self.assertTrue(analyzer.contains_codeblocks([varied_spaces]))
        self.assertFalse(analyzer.contains_codeblocks([" " * 8]))

        self.assertFalse(analyzer.contains_codeblocks([""]))

    def test_categorize_tags(self):
        title = "This is a sample post (volume 1) {part 2} |part 3|"
        analyzer = PostAnalyzer()
        series, final, invalid = analyzer.categorize_tags(title)
        self.assertEqual(len(invalid), 0, f"Unexpected bad tags: {invalid}")
        self.assertTrue(series)
        self.assertFalse(final)

    def test_mixed_series_final(self):
{'fraction_non_alphanumeric': 0.10513310120184208, 'fraction_numerical': 0.005166797708637538, 'mean_word_length': 3.0787906550618414, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 8, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4790764', 'n_tokens_mistral': 2695, 'n_tokens_neox': 2620, 'n_words': 621}