<commit_before>import re
import requests
from bs4 import BeautifulSoup


def get_content(url):

    paste = {
        'site': 'prntscr',
        'url':  url,
    }
    response = requests.get(url)
    if response.status_code != 200:
        return

    data = response.text
    soup = BeautifulSoup(data)
    url = soup.find('meta', {'property': 'og:image'})['content']

    param_str_raw = url[url.index('?')+1:]
    params = {k: v for (k, v) in (
        kv.split('=') for kv in param_str_raw.split('&'))}

    m = re.match('^.*/([a-zA-Z0-9]+)\.([a-zA-Z]+)$', url)
    response = requests.get(params['url'])
    if response.status_code != 200 or not m:
        return

    paste['content'] = response.content
    paste['orig_filename'] = m.group(1)
    paste['ext'] = m.group(2)
    return [paste]<commit_msg>Added html5lib parser param to prntscrn module<commit_after>import re
import requests
from bs4 import BeautifulSoup


def get_content(url):

    paste = {
        'site': 'prntscr',
        'url':  url,
    }
    response = requests.get(url)
    if response.status_code != 200:
        return

    data = response.text
    soup = BeautifulSoup(data, "html5lib")
    url = soup.find('meta', {'property': 'og:image'})['content']

    param_str_raw = url[url.index('?')+1:]
    params = {k: v for (k, v) in (
        kv.split('=') for kv in param_str_raw.split('&'))}

    m = re.match('^.*/([a-zA-Z0-9]+)\.([a-zA-Z]+)$', url)
    response = requests.get(params['url'])
    if response.status_code != 200 or not m:
        return

    paste['content'] = response.content
    paste['orig_filename'] = m.group(1)
    paste['ext'] = m.group(2)
    return [paste]
{'fraction_non_alphanumeric': 0.1793641271745651, 'fraction_numerical': 0.015596880623875225, 'mean_word_length': 2.8433179723502304, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7223944', 'n_tokens_mistral': 616, 'n_tokens_neox': 583, 'n_words': 132}