<commit_before># -*- coding: utf-8 -*-
import json
import praw
import prawcore
import codecs
import requests
import urllib
import configparser
import time
import traceback
import sys

config_parser = configparser.ConfigParser()
config_parser.read('praw.ini')

config = config_parser['config']
user_agent = config_parser['hunter']['user_agent']

user_to_search = config['user_to_search']
user_to_pm = config['user_to_pm']
posts_to_search = config['posts_to_search']
delete_posts = config['delete_posts']

reddit = praw.Reddit('hunter')
reddit.user.me()


def is_username_available(username):
    endpoint = 'https://www.reddit.com/api/username_available.json?user='
    url = endpoint + username
    response = requests.get(url)
    return response.content == 'True'


def get_submissions(username, limit=0, before='', sort='new'):
    if not before.startswith('t3_'):
        before = ''
    endpoint = 'https://www.reddit.com/user/{}/submitted.json?sort={}&limit={}&before={}'
    url = endpoint.format(username, sort, limit, before)
    # print(url)
    request = urllib.request.Request(
        url,
        data=None,
        headers={
            'User-Agent': user_agent
        }
    )
    response = urllib.request.urlopen(request)
    return response


def get_spam_user(title):
    import re
    str_to_replace = 'Overview for '
    start_with = re.match(str_to_replace, title, re.I)
    if not start_with:
        return

    str_replaced = ''
    pattern = re.compile(str_to_replace, re.IGNORECASE)
    username = pattern.sub(str_replaced, title)
    return username


def get_user_status(username):
    time.sleep(1)
    try:
        redditor = reddit.redditor(username)
        if getattr(redditor, 'is_suspended', False):
            return 'suspended'  # account is suspended
    except prawcore.NotFound:
        if is_username_available(username):
            return 'not_exists'  # account doesn't exist
        else:
            return 'banned'  # account is deleted or shadowbanned for spam
    else:
        return 'exists'  # account exists


def get_spam_posts(username, sub_to_search='spam', limit=10):
    response = get_submissions(username, limit=limit)
    reader = codecs.getreader("utf-8")
    json_dic = json.load(reader(response))
    childrens = json_dic['data']['children']

    active = []
    banned = []
    for children in childrens:
        subreddit = children['data']['subreddit']

        if subreddit == sub_to_search:
            title = children['data']['title']

            username = get_spam_user(title)
            if not username:
                continue
            print('Spam post found for user: ' + username)
            try:
                result = get_user_status(username)
                id = children['data']['id']

                data = []
                data.append(id)
                data.append(username)

                # print(username + ' ' + result)
                if result == 'banned':
                    banned.append(data)
                else:
                    active.append(data)
            except Exception:
                pass

    return banned, active


def to_url(text, url):
    return '[' + text + '](' + url + ')'

<commit_msg>Ignore error for now.<commit_after># -*- coding: utf-8 -*-
import json
import praw
import prawcore
import codecs
import requests
import urllib
import configparser
import time
import traceback
import sys

config_parser = configparser.ConfigParser()
config_parser.read('praw.ini')

config = config_parser['config']
user_agent = config_parser['hunter']['user_agent']

user_to_search = config['user_to_search']
user_to_pm = config['user_to_pm']
posts_to_search = config['posts_to_search']
delete_posts = config['delete_posts']

reddit = praw.Reddit('hunter')
reddit.user.me()


def is_username_available(username):
    endpoint = 'https://www.reddit.com/api/username_available.json?user='
    url = endpoint + username
    response = requests.get(url)
    return response.content == 'True'


def get_submissions(username, limit=0, before='', sort='new'):
    if not before.startswith('t3_'):
        before = ''
    endpoint = 'https://www.reddit.com/user/{}/submitted.json?sort={}&limit={}&before={}'
    url = endpoint.format(username, sort, limit, before)
    # print(url)
    request = urllib.request.Request(
        url,
        data=None,
        headers={
            'User-Agent': user_agent
        }
    )
    response = urllib.request.urlopen(request)
    return response


def get_spam_user(title):
    import re
    str_to_replace = 'Overview for '
    start_with = re.match(str_to_replace, title, re.I)
    if not start_with:
        return

    str_replaced = ''
    pattern = re.compile(str_to_replace, re.IGNORECASE)
    username = pattern.sub(str_replaced, title)
    return username


def get_user_status(username):
    time.sleep(1)
    try:
        redditor = reddit.redditor(username)
        if getattr(redditor, 'is_suspended', False):
            return 'suspended'  # account is suspended
    except prawcore.NotFound:
        if is_username_available(username):
            return 'not_exists'  # account doesn't exist
        else:
            return 'banned'  # account is deleted or shadowbanned for spam
    else:
        return 'exists'  # account exists


def get_spam_posts(username, sub_to_search='spam', limit=10):
    response = get_submissions(username, limit=limit)
    reader = codecs.getreader("utf-8")
    json_dic = json.load(reader(response))
    childrens = json_dic['data']['children']

    active = []
    banned = []
    for children in childrens:
        subreddit = children['data']['subreddit']

        if subreddit == sub_to_search:
            title = children['data']['title']

            username = get_spam_user(title)
            if not username:
                continue
            try:
                result = get_user_status(username)
                print('Spam post found for user: ' + username)
                id = children['data']['id']

                data = []
                data.append(id)
                data.append(username)

                if result == 'banned':
                    banned.append(data)
                else:
                    active.append(data)

            except Exception:
                pass

    return banned, active


def to_url(text, url):
    return '[' + text + '](' + url + ')'


{'fraction_non_alphanumeric': 0.11509669369931379, 'fraction_numerical': 0.002183406113537118, 'mean_word_length': 2.545052515201769, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 4, 'lorem ipsum': 0, 'www.': 4, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '850980', 'n_tokens_mistral': 1947, 'n_tokens_neox': 1900, 'n_words': 421}