<commit_before>import logging

from google.appengine.dist import use_library
use_library('django', '1.2')

appstats_RECORD_FRACTION = .1

class BlockingMiddleware(object):
    def __init__(self, app):
        self._wrapped_app = app

    def __call__(self, environ, start_response):
        user_agent = environ.get('HTTP_USER_AGENT', '')
        # Scraper running on EC2 (IP addresses 192.168.127.12 and 172.16.17.32)
        # that's requesting lots of Tweet Digest pages.
        if user_agent == 'Python-urllib/2.7':
            logging.info('Blocked request')
            start_response('403 Forbidden', [('Content-type','text/plain')])
            return ['']

        # Google Reader and iGoogle are dead, yet their crawler still lives on.
        # (and generates ~1,200 requests per day). Block it, since presumably no
        # one is actually looking at the resuts.
        if user_agent.startswith('Feedfetcher-Google;'):
            logging.info('Blocked Feedfetcher request')
            start_response('403 Forbidden', [('Content-type','text/plain')])
            return ['']

        return self._wrapped_app(environ, start_response)

def webapp_add_wsgi_middleware(app):
    from google.appengine.ext.appstats import recording
    app = recording.appstats_wsgi_middleware(app)

    app = BlockingMiddleware(app)

    return app
<commit_msg>Give a 200 response to Feedfetcher so that it can be cached.<commit_after>import logging

from google.appengine.dist import use_library
use_library('django', '1.2')

appstats_RECORD_FRACTION = .1

class BlockingMiddleware(object):
    def __init__(self, app):
        self._wrapped_app = app

    def __call__(self, environ, start_response):
        user_agent = environ.get('HTTP_USER_AGENT', '')
        # Scraper running on EC2 (IP addresses 192.168.127.12 and 172.16.17.32)
        # that's requesting lots of Tweet Digest pages.
        if user_agent == 'Python-urllib/2.7':
            logging.info('Blocked request')
            start_response('403 Forbidden', [('Content-type','text/plain')])
            return ['']

        # Google Reader and iGoogle are dead, yet their crawler still lives on.
        # (and generates ~1,200 requests per day). Block it, since presumably no
        # one is actually looking at the resuts. Make the block be a 200
        # response, so that it gets cached by App Engine's edge cache (and
        # hopefully Trawler too).
        if user_agent.startswith('Feedfetcher-Google;'):
            logging.info('Blocked Feedfetcher request')
            start_response('200 OK', [
                ('Content-type','text/xml; charset=UTF-8'),
                ('Last-Modified', 'Tue, 2 Jul 2013 00:00:00 GMT'),
                ('Expires', 'Sun, 2 Jul 2023 00:00:00 GMT'),
                ('Cache-Control', 'public, max-age=315570000'),
            ])
            return ['''<?xml version="1.0" encoding="utf-8"?>
<feed xmlns="http://www.w3.org/2005/Atom">
  <id>tag:streamspigot.com,2013:/feedfetcher-go-away</id>
  <title>Stream Spigot</title>
  <entry>
    <id>tag:streamspigot.com,2013:/feedfetcher-go-away</id>
    <title type="text">Feedfetcher, why are you still crawling?</title>
    <content type="xhtml">
      <div xmlns="http://www.w3.org/1999/xhtml">
        Since <a href="http://googlereader.blogspot.com/2013/07/a-final-farewell.html">Reader</a> and <a href="https://support.google.com/websearch/answer/2664197?hl=en">iGoogle</a> are shut down, why is <a href="http://www.google.com/feedfetcher.html">Feedfetcher</a> still crawling?
      </div>
    </content>
  </entry>
</feed>
''']
        return self._wrapped_app(environ, start_response)

def webapp_add_wsgi_middleware(app):
    from google.appengine.ext.appstats import recording
    app = recording.appstats_wsgi_middleware(app)

    app = BlockingMiddleware(app)

    return app

{'fraction_non_alphanumeric': 0.1271120353522225, 'fraction_numerical': 0.03743176501169743, 'mean_word_length': 3.0167014613778704, 'pattern_counts': {'":': 0, '<': 26, '<?xml version=': 1, '>': 26, 'https://': 1, 'lorem ipsum': 0, 'www.': 3, 'xml': 4}, 'pii_count': 4, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '478147', 'n_tokens_mistral': 1303, 'n_tokens_neox': 1181, 'n_words': 294}