<commit_before># A source from which we fetch captcha images
# TODO: interactive test

import requests
from io import BytesIO
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import captcha_recognizer

captcha_length = 5
#       'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
chars = '    EFGH JKLMN PQR TUVWXY  123456 89'.replace(' ', '')
charset = set(chars)
session = None
cookie = None

def _get_captcha_url(use_https=False):
    return ('https' if use_https else 'http') + '://www.bilibili.com/captcha'


def canonicalize(seq):
    return seq.upper()


def fetch_image(use_https=False, retry_limit=3):
    global session
    global cookie
    session = requests.session()
    url = _get_captcha_url(use_https)
    header = {
        'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",
    }
    print('Fetching CAPTCHA image from {}'.format(url))
    r = None
    for num_retries in range(retry_limit):
        if num_retries > 0:
            print('num_retries = {}'.format(num_retries))
        try:
            r = session.get(url, headers=header)
            cookie = r.request.headers['cookie']
            break
        except Exception as e:
            print(e)
    return None if r is None else mpimg.imread(BytesIO(r.content))


# TODO: Bugs, probably need header and cookie
def fill_captcha(use_https=False):
    image = fetch_image(use_https)
    if session is None or cookie is None:
        raise ValueError('No session or session id found')
    plt.ion()
    plt.show()
    plt.clf()
    plt.axis('off')
    plt.imshow(image)
    plt.show()
    plt.pause(1e-2)
    while True:
        captcha = input('Enter the char sequence: ')
        if len(captcha) == captcha_length:
            break
    plt.ioff()
    #captcha = captcha_recognizer.CaptchaRecognizer().recognize(image)
    url = 'https' if use_https else 'http' + '://account.bilibili.com/register/mail'
    header = {
        'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",
        'Host': "account.bilibili.com",
        'Referer': url,
        'Cookie': cookie,
        'Origin': 'http://account.bilibili.com'
    }
    data = {'vd': captcha, 'action': "checkVd"}
    print(cookie)
    r = session.post(url, headers=header, data=data)
    print(r.json())
    if r.json()['status'] == 'false':
        return False
    elif r.json()['status'] == 'True':
        return True<commit_msg>Note: non HTTPS always fails, even in a browser<commit_after># A source from which we fetch captcha images
# TODO: interactive test

import requests
from io import BytesIO
import matplotlib.image as mpimg
import matplotlib.pyplot as plt
import captcha_recognizer

captcha_length = 5
#       'ABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789'
chars = '    EFGH JKLMN PQR TUVWXY  123456 89'.replace(' ', '')
charset = set(chars)
session = None
cookie = None


def _get_captcha_url(use_https=False):
    return ('https' if use_https else 'http') + '://www.bilibili.com/captcha'


def canonicalize(seq):
    return seq.upper()


def fetch_image(use_https=False, retry_limit=3):
    global session
    global cookie
    session = requests.session()
    url = _get_captcha_url(use_https)
    header = {
        'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",
    }
    print('Fetching CAPTCHA image from {}'.format(url))
    r = None
    for num_retries in range(retry_limit):
        if num_retries > 0:
            print('num_retries = {}'.format(num_retries))
        try:
            r = session.get(url, headers=header)
            # print(r.request.headers)
            # cookie = r.request.headers['cookie']
            break
        except Exception as e:
            print(e)
    return None if r is None else mpimg.imread(BytesIO(r.content))


# TODO: Bugs, probably need header and cookie
def fill_captcha(use_https=True):
    image = fetch_image(use_https)
    if session is None:# or cookie is None:
        raise ValueError('No session or session id found')
    plt.ion()
    plt.show()
    plt.clf()
    plt.axis('off')
    plt.imshow(image)
    plt.show()
    plt.pause(1e-2)
    while True:
        captcha = input('Enter the char sequence: ')
        if len(captcha) == captcha_length:
            break
    plt.ioff()
    #captcha = captcha_recognizer.CaptchaRecognizer().recognize(image)
    url = ('https' if use_https else 'http') + '://account.bilibili.com/register/mail'
    header = {
        'User-Agent': "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:34.0) Gecko/20100101 Firefox/34.0",
        'Host': "account.bilibili.com",
        'Referer': url,
        # 'Cookie': cookie,
        'Origin': ('https' if use_https else 'http') +  '://account.bilibili.com'
    }
    data = {'vd': captcha, 'action': "checkVd"}
    # print(cookie)
    r = session.post(url, headers=header, data=data)
    print(r.request.headers)
    print(data)
    print(r.json())
    if r.json()['status'] == 'false':
        return False
    elif r.json()['status'] == 'True':
        return True
{'fraction_non_alphanumeric': 0.12804878048780488, 'fraction_numerical': 0.026357199055861525, 'mean_word_length': 2.8581183611532626, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2176665', 'n_tokens_mistral': 1827, 'n_tokens_neox': 1706, 'n_words': 429}