<commit_before>import re
import requests
import lxml.html

        
def grab_cloudflare(url, *args, **kwargs):
    sess = requests.Session()
    sess.headers = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0"}
    safe_eval = lambda s: eval(s, {"__builtins__": {}}) if "#" not in s and "__" not in s else ""
    page = sess.get(url, *args, **kwargs).content

    if "a = document.getElementById('jschl-answer');" in page:
        logger.info("Encountered CloudFlare anti-bot wall")
        # Cloudflare anti-bots is on
        html = lxml.html.fromstring(page.content)
        challenge = html.find(".//input[@name='jschl_vc']").attrib["value"]
        script = html.findall(".//script")[-1].text_content()
        domain_parts = url.split("/")
        domain = domain_parts[2]
        math = re.search(r"a\.value = (\d.+?);", script).group(1)
        answer = str(safe_eval(math) + len(domain))
        data = {"jschl_vc": challenge, "jschl_answer": answer}
        get_url = domain_parts[0] + '//' + domain + "/cdn-cgi/l/chk_jschl"
        return sess.get(get_url, params=data, headers={'referer': url}, *args, **kwargs).content
    else:
        return page<commit_msg>Update cfscrape.py<commit_after>import re
import requests
import lxml.html

        
def grab_cloudflare(url, *args, **kwargs):
    sess = requests.Session()
    sess.headers = {"User-Agent": "Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:17.0) Gecko/20100101 Firefox/17.0"}
    safe_eval = lambda s: eval(s, {"__builtins__": {}}) if "#" not in s and "__" not in s else ""
    page = sess.get(url, *args, **kwargs).content

    if "a = document.getElementById('jschl-answer');" in page:
        # Cloudflare anti-bots is on
        html = lxml.html.fromstring(page)
        challenge = html.find(".//input[@name='jschl_vc']").attrib["value"]
        script = html.findall(".//script")[-1].text_content()
        domain_parts = url.split("/")
        domain = domain_parts[2]
        math = re.search(r"a\.value = (\d.+?);", script).group(1)
        answer = str(safe_eval(math) + len(domain))
        data = {"jschl_vc": challenge, "jschl_answer": answer}
        get_url = domain_parts[0] + '//' + domain + "/cdn-cgi/l/chk_jschl"
        return sess.get(get_url, params=data, headers={"Referer": url}, *args, **kwargs).content
    else:
        return page
{'fraction_non_alphanumeric': 0.17460986925347954, 'fraction_numerical': 0.021931674398987768, 'mean_word_length': 3.04778156996587, 'pattern_counts': {'":': 9, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 4}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4684404', 'n_tokens_mistral': 860, 'n_tokens_neox': 814, 'n_words': 174}