<commit_before># vim: set sw=4 et:

import json
import logging
import brozzler
import reppy.cache
import requests

__all__ = ["is_permitted_by_robots"]

_robots_caches = {}  # {site_id:reppy.cache.RobotsCache}
def _robots_cache(site):
    class SessionRaiseOn420(requests.Session):
        def get(self, url, *args, **kwargs):
            res = super().get(url, *args, **kwargs)
            if res.status_code == 420 and 'warcprox-meta' in res.headers:
                raise brozzler.ReachedLimit(warcprox_meta=json.loads(res.headers['warcprox-meta']), http_payload=res.text)
            else:
                return res

    if not site.id in _robots_caches:
        req_sesh = SessionRaiseOn420()
        req_sesh.verify = False   # ignore cert errors
        if site.proxy:
            proxie = "http://{}".format(site.proxy)
            req_sesh.proxies = {"http":proxie,"https":proxie}
        if site.extra_headers:
            req_sesh.headers.update(site.extra_headers)
        _robots_caches[site.id] = reppy.cache.RobotsCache(session=req_sesh)

    return _robots_caches[site.id]

def is_permitted_by_robots(site, url):
    if site.ignore_robots:
        return True
    try:
        result = _robots_cache(site).allowed(url, "brozzler")
        return result
    except BaseException as e:
        if isinstance(e, reppy.exceptions.ServerError) and isinstance(e.args[0], brozzler.ReachedLimit):
            raise e.args[0]
        else:
            logging.error("problem with robots.txt for %s: %s", url, repr(e), exc_info=True)
            return False
<commit_msg>since I can't figure out what's causing these sporadic errors fetching certain robots.txt through warcprox, stick a retry loop around the fetch<commit_after># vim: set sw=4 et:

import json
import logging
import brozzler
import reppy.cache
import requests

__all__ = ["is_permitted_by_robots"]

_robots_caches = {}  # {site_id:reppy.cache.RobotsCache}
def _robots_cache(site):
    class SessionRaiseOn420(requests.Session):
        def get(self, url, *args, **kwargs):
            res = super().get(url, *args, **kwargs)
            if res.status_code == 420 and 'warcprox-meta' in res.headers:
                raise brozzler.ReachedLimit(warcprox_meta=json.loads(res.headers['warcprox-meta']), http_payload=res.text)
            else:
                return res

    if not site.id in _robots_caches:
        req_sesh = SessionRaiseOn420()
        req_sesh.verify = False   # ignore cert errors
        if site.proxy:
            proxie = "http://{}".format(site.proxy)
            req_sesh.proxies = {"http":proxie,"https":proxie}
        if site.extra_headers:
            req_sesh.headers.update(site.extra_headers)
        _robots_caches[site.id] = reppy.cache.RobotsCache(session=req_sesh)

    return _robots_caches[site.id]

def is_permitted_by_robots(site, url):
    if site.ignore_robots:
        return True

    tries_left = 10
    while True:
        try:
            result = _robots_cache(site).allowed(url, "brozzler")
            return result
        except BaseException as e:
            if isinstance(e, reppy.exceptions.ServerError) and isinstance(e.args[0], brozzler.ReachedLimit):
                raise e.args[0]
            else:
                if tries_left > 0:
                    logging.warn("caught exception %s fetching robots.txt (%s tries left) for %s", repr(e), tries_left, url)
                    tries_left -= 1
                else:
                    logging.error("caught exception fetching robots.txt (0 tries left) for %s: %s", url, repr(e), exc_info=True)
                    return False

{'fraction_non_alphanumeric': 0.10489897592028785, 'fraction_numerical': 0.008026570716855799, 'mean_word_length': 2.5466143277723257, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3781450', 'n_tokens_mistral': 1201, 'n_tokens_neox': 1149, 'n_words': 262}