<commit_before>'''
Stuff related to robots.txt processing
'''

import asyncio

import time
import json
import logging

import robotexclusionrulesparser
import magic

import stats
import fetcher

LOGGER = logging.getLogger(__name__)

class Robots:
    def __init__(self, robotname, session, datalayer, config):
        self.robotname = robotname
        self.session = session
        self.datalayer = datalayer
        self.config = config
        self.rerp = robotexclusionrulesparser.RobotExclusionRulesParser()
        self.max_tries = self.config.get('Robots', {}).get('MaxTries')
        self.in_progress = set()
        self.magic = magic.Magic(flags=magic.MAGIC_MIME_TYPE)
        self.jsonlogfile = self.config.get('Logging', {}).get('Robotslog')
        if self.jsonlogfile:
            self.jsonlogfd = open(self.jsonlogfile, 'w')
        else:
            self.jsonlogfd = None

    async def check(self, url, parts, headers=None, proxy=None, mock_robots=None):
        schemenetloc = parts.scheme + '://' + parts.netloc
        if parts.path:
            pathplus = parts.path
        else:
            pathplus = '/'
        if parts.params:
            pathplus += ';' + parts.params
        if parts.query:
            pathplus += '?' + parts.query

        try:
            robots = self.datalayer.read_robots_cache(schemenetloc)
        except KeyError:
            # extra semantics and policy inside fetch_robots: 404 returns '', etc. also inserts into cache.
            robots = await self.fetch_robots(schemenetloc, mock_robots, headers=headers, proxy=proxy)

        if robots is None:
            self.jsonlog(schemenetloc, {'error':'unable to find robots information', 'action':'deny'})
            stats.stats_sum('robots denied', 1)
            return False

        if len(robots) == 0:
            return True

        start = time.clock()
        self.rerp.parse(robots) # cache this parse?
        stats.record_cpu_burn('robots parse', start)

#        if self.rerp.sitemaps:
#           ...

        start = time.clock()
        check = self.rerp.is_allowed(self.robotname, pathplus)
        stats.record_cpu_burn('robots is_allowed', start)

        if check:
            # don't log success
            return True

        self.jsonlog(schemenetloc, {'url':pathplus, 'action':'deny'})
        stats.stats_sum('robots denied', 1)
        return False

    async def fetch_robots(self, schemenetloc, mock_url, headers=None, proxy=None):
        '''
        robotexclusionrules parser is not async, so fetch the file ourselves
        '''
        url = schemenetloc + '/robots.txt'

        if proxy:
            raise ValueError('not yet implemented')

        # We might enter this routine multiple times, so, sleep if we aren't the first
        # XXX this is frequently racy, according to the logfiles!
        if schemenetloc in self.in_progress:
            while schemenetloc in self.in_progress:
                # XXX make this a stat?
                # XXX does it go off for wide when it shouldn't?
                LOGGER.debug('sleeping because someone beat me to the robots punch')
                await asyncio.sleep(0.3)

            # at this point robots might be in the cache... or not.
            try:
                robots = self.datalayer.read_robots_cache(schemenetloc)
            except KeyError:
                robots = None
            if robots is not None:
                return robots

            # ok, so it's not in the cache -- and the other guy's
            # fetch failed. if we just fell through there would be a
            # big race. treat this as a failure.
            # XXX note that we have no negative caching
            LOGGER.debug('some other fetch of robots has failed.') # XXX make this a stat
            return None

        self.in_progress.add(schemenetloc)

        response, body_bytes, header_bytes, apparent_elapsed, last_exception = await fetcher.fetch(
            url, self.session, self.config, headers=headers, proxy=proxy,
            mock_url=mock_url, allow_redirects=True
        )

        if last_exception:
            self.jsonlog(schemenetloc, {'error':'max tries exceeded, final exception is: ' + last_exception,
                                        'action':'fetch'})
            self.in_progress.discard(schemenetloc)
            return None

        # if we got a 404, return an empty robots.txt
        if response.status == 404:
            self.jsonlog(schemenetloc, {'error':'got a 404, treating as empty robots',
                                        'action':'fetch', 'apparent_elapsed':apparent_elapsed})
            self.datalayer.cache_robots(schemenetloc, '')
            self.in_progress.discard(schemenetloc)
            await response.release()
            return ''

        # if we got a non-200, some should be empty and some should be None (XXX Policy)
        if str(response.status).startswith('4') or str(response.status).startswith('5'):
            self.jsonlog(schemenetloc,
                         {'error':'got an unexpected status of {}, treating as deny'.format(response.status),
                          'action':'fetch', 'apparent_elapsed':apparent_elapsed})
            self.in_progress.discard(schemenetloc)
            await response.release()
            return None

        if not self.is_plausible_robots(schemenetloc, body_bytes, apparent_elapsed):
            # treat as empty
            self.datalayer.cache_robots(schemenetloc, '')
            self.in_progress.discard(schemenetloc)
            await response.release()
            return ''
<commit_msg>log these<commit_after>'''
Stuff related to robots.txt processing
'''

import asyncio

import time
import json
import logging

import robotexclusionrulesparser
import magic

import stats
import fetcher

LOGGER = logging.getLogger(__name__)

class Robots:
    def __init__(self, robotname, session, datalayer, config):
        self.robotname = robotname
        self.session = session
        self.datalayer = datalayer
        self.config = config
        self.rerp = robotexclusionrulesparser.RobotExclusionRulesParser()
        self.max_tries = self.config.get('Robots', {}).get('MaxTries')
        self.in_progress = set()
        self.magic = magic.Magic(flags=magic.MAGIC_MIME_TYPE)
        self.jsonlogfile = self.config.get('Logging', {}).get('Robotslog')
        if self.jsonlogfile:
            self.jsonlogfd = open(self.jsonlogfile, 'w')
        else:
            self.jsonlogfd = None

    async def check(self, url, parts, headers=None, proxy=None, mock_robots=None):
        schemenetloc = parts.scheme + '://' + parts.netloc
        if parts.path:
            pathplus = parts.path
        else:
            pathplus = '/'
        if parts.params:
            pathplus += ';' + parts.params
        if parts.query:
            pathplus += '?' + parts.query

        try:
            robots = self.datalayer.read_robots_cache(schemenetloc)
        except KeyError:
            # extra semantics and policy inside fetch_robots: 404 returns '', etc. also inserts into cache.
            robots = await self.fetch_robots(schemenetloc, mock_robots, headers=headers, proxy=proxy)

        if robots is None:
            self.jsonlog(schemenetloc, {'error':'unable to find robots information', 'action':'deny'})
            stats.stats_sum('robots denied', 1)
            return False

        if len(robots) == 0:
            return True

        start = time.clock()
        self.rerp.parse(robots) # cache this parse?
        stats.record_cpu_burn('robots parse', start)

#        if self.rerp.sitemaps:
#           ...

        start = time.clock()
        check = self.rerp.is_allowed(self.robotname, pathplus)
        stats.record_cpu_burn('robots is_allowed', start)

        if check:
            # don't log success
            return True

        self.jsonlog(schemenetloc, {'url':pathplus, 'action':'deny'})
        stats.stats_sum('robots denied', 1)
        return False

    async def fetch_robots(self, schemenetloc, mock_url, headers=None, proxy=None):
        '''
        robotexclusionrules parser is not async, so fetch the file ourselves
        '''
        url = schemenetloc + '/robots.txt'

        if proxy:
            raise ValueError('not yet implemented')

        # We might enter this routine multiple times, so, sleep if we aren't the first
        # XXX this is frequently racy, according to the logfiles!
        if schemenetloc in self.in_progress:
            while schemenetloc in self.in_progress:
                # XXX make this a stat?
                # XXX does it go off for wide when it shouldn't?
                LOGGER.debug('sleeping because someone beat me to the robots punch')
                await asyncio.sleep(0.3)

            # at this point robots might be in the cache... or not.
            try:
                robots = self.datalayer.read_robots_cache(schemenetloc)
            except KeyError:
                robots = None
            if robots is not None:
                return robots

            # ok, so it's not in the cache -- and the other guy's
            # fetch failed. if we just fell through there would be a
            # big race. treat this as a failure.
            # XXX note that we have no negative caching
            LOGGER.debug('some other fetch of robots has failed.') # XXX make this a stat
            return None

        self.in_progress.add(schemenetloc)

        response, body_bytes, header_bytes, apparent_elapsed, last_exception = await fetcher.fetch(
            url, self.session, self.config, headers=headers, proxy=proxy,
            mock_url=mock_url, allow_redirects=True
        )

        if last_exception:
            self.jsonlog(schemenetloc, {'error':'max tries exceeded, final exception is: ' + last_exception,
                                        'action':'fetch'})
            self.in_progress.discard(schemenetloc)
            return None

        # if we got a 404, return an empty robots.txt
        if response.status == 404:
            self.jsonlog(schemenetloc, {'error':'got a 404, treating as empty robots',
                                        'action':'fetch', 'apparent_elapsed':apparent_elapsed})
            self.datalayer.cache_robots(schemenetloc, '')
            self.in_progress.discard(schemenetloc)
            await response.release()
            return ''

        # if we got a non-200, some should be empty and some should be None (XXX Policy)
        if str(response.status).startswith('4') or str(response.status).startswith('5'):
            self.jsonlog(schemenetloc,
                         {'error':'got an unexpected status of {}, treating as deny'.format(response.status),
                          'action':'fetch', 'apparent_elapsed':apparent_elapsed})
            self.in_progress.discard(schemenetloc)
            await response.release()
            return None

        if not self.is_plausible_robots(schemenetloc, body_bytes, apparent_elapsed):
            # policy: treat as empty
            self.jsonlog(schemenetloc,
                         {'warning':'saw an implausible robots.txt, treating as empty',
                          'action':'fetch', 'apparent_elapsed':apparent_elapsed})
            self.datalayer.cache_robots(schemenetloc, '')
            self.in_progress.discard(schemenetloc)
            await response.release()
            return ''

{'fraction_non_alphanumeric': 0.09212481426448738, 'fraction_numerical': 0.003845817673280308, 'mean_word_length': 2.05445808862787, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4076357', 'n_tokens_mistral': 3252, 'n_tokens_neox': 3110, 'n_words': 868}