<commit_before># -*- coding: utf-8 -*-
import regex
import phonenumbers
from bs4 import BeautifulSoup


def all_caps_title(s, site):
    if regex.compile(ur"SQL|\b(ERROR|PHP|QUERY|ANDROID|CASE|SELECT|HAVING|COUNT)\b").search(s):
        return False   # common words in non-spam all-caps titles
    return bool(regex.compile(ur"^(?=.*\p{upper})\P{lower}*$", regex.UNICODE).search(s))


def has_repeated_words(s, site):
    words = regex.split(r"[\s.,;!/\()\[\]+_-]", s)
    words = [w for w in words if w != ""]
    curr = 0
    prev = ""
    for w in words:
        if w == prev and w.isalpha() and len(w) > 1:
            curr += 1
        else:
            curr = 0
        prev = w
        if curr >= 5 and curr * len(w) >= 0.1 * len(s):
            return True<commit_msg>rm BeautifulSoup<commit_after># -*- coding: utf-8 -*-
import regex
import phonenumbers


def all_caps_title(s, site):
    if regex.compile(ur"SQL|\b(ERROR|PHP|QUERY|ANDROID|CASE|SELECT|HAVING|COUNT)\b").search(s):
        return False   # common words in non-spam all-caps titles
    return bool(regex.compile(ur"^(?=.*\p{upper})\P{lower}*$", regex.UNICODE).search(s))


def has_repeated_words(s, site):
    words = regex.split(r"[\s.,;!/\()\[\]+_-]", s)
    words = [w for w in words if w != ""]
    curr = 0
    prev = ""
    for w in words:
        if w == prev and w.isalpha() and len(w) > 1:
            curr += 1
        else:
            curr = 0
        prev = w
        if curr >= 5 and curr * len(w) >= 0.1 * len(s):
            return True
{'fraction_non_alphanumeric': 0.1741977734119188, 'fraction_numerical': 0.011132940406024885, 'mean_word_length': 2.3656387665198237, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '703238', 'n_tokens_mistral': 567, 'n_tokens_neox': 547, 'n_words': 142}