<commit_before># Portions Copyright (c) 2010 OpenStack, LLC.
# Everything else Copyright (c) 2011 Wikimedia Foundation, Inc.
# all of it licensed under the Apache Software License, included by reference.

# unit test is in test_rewrite.py. Tests are referenced by numbered comments.

import webob
import webob.exc
import re
import eventlet
from eventlet.green import urllib2
import time
import urlparse
from swift.common.utils import get_logger
from swift.common.wsgi import WSGIContext


class DumbRedirectHandler(urllib2.HTTPRedirectHandler):
    def http_error_301(self, req, fp, code, msg, headers):
        return None

    def http_error_302(self, req, fp, code, msg, headers):
        return None


class _WMFRewriteContext(WSGIContext):
    """
    Rewrite Media Store URLs so that swift knows how to deal with them.
    """

    def __init__(self, rewrite, conf):
        WSGIContext.__init__(self, rewrite.app)
        self.app = rewrite.app
        self.logger = rewrite.logger

        self.account = conf['account'].strip()
        self.thumbhost = conf['thumbhost'].strip()
        self.thumborhost = conf['thumborhost'].strip() if 'thumborhost' in conf else None
        self.thumbor_wiki_list = [item.strip() for item in conf['thumbor_wiki_list'].split(',')] if 'thumbor_wiki_list' in conf else None
        self.user_agent = conf['user_agent'].strip()
        self.bind_port = conf['bind_port'].strip()
        self.shard_container_list = [item.strip() for item in conf['shard_container_list'].split(',')]
        # this parameter controls whether URLs sent to the thumbhost are sent as is (eg. upload/proj/lang/) or with the site/lang
        # converted  and only the path sent back (eg en.wikipedia/thumb).
        self.backend_url_format = conf['backend_url_format'].strip()  # asis, sitelang

    def collectHttpStatusCodes(self, url, thumbor_thread, mediawiki_code):
        self.logger.debug("Mediawiki: %d %s" % (mediawiki_code, url))

        if thumbor_thread is None:
            return

        try:
            # Waits for Thumbor if it took longer than Mediawiki to process the image
            # Otherwise returns/throws exceptions immediately
            thumbor_result = thumbor_thread.wait()
            code = thumbor_result.getcode()
        except urllib2.HTTPError, error:
            code = error.code
        except urllib2.URLError, error:
            code = 503

        self.logger.debug("Thumbor: %d %s" % (code, url))

        if code != mediawiki_code:
            self.logger.warn("HTTP status code mismatch. Mediawiki: %d Thumbor: %d URL: %s" % (mediawiki_code, code, url))

    def handle404(self, reqorig, url, container, obj):
        """
        Return a webob.Response which fetches the thumbnail from the thumb
        host and returns it. Note also that the thumb host might write it out
        to Swift so it won't 404 next time.
        """
        original_request_url = reqorig.url
        # go to the thumb media store for unknown files
        reqorig.host = self.thumbhost
        # upload doesn't like our User-agent, otherwise we could call it
        # using urllib2.url()
        proxy_handler = urllib2.ProxyHandler({'http': self.thumbhost})
        redirect_handler = DumbRedirectHandler()

        opener = urllib2.build_opener(redirect_handler, proxy_handler)
        # Thumbor doesn't need (and doesn't like) the proxy
        thumbor_opener = urllib2.build_opener(redirect_handler)

        # Pass on certain headers from the caller squid to the scalers
        opener.addheaders = []
        if reqorig.headers.get('User-Agent') is not None:
            opener.addheaders.append(('User-Agent', reqorig.headers.get('User-Agent')))
        else:
            opener.addheaders.append(('User-Agent', self.user_agent))
        for header_to_pass in ['X-Forwarded-For', 'X-Forwarded-Proto',
                               'Accept', 'Accept-Encoding', 'X-Original-URI']:
            if reqorig.headers.get(header_to_pass) is not None:
                opener.addheaders.append((header_to_pass, reqorig.headers.get(header_to_pass)))

        thumbor_opener.addheaders = opener.addheaders

        self.logger.debug("Addheaders: %r" % thumbor_opener.addheaders)

        # At least in theory, we shouldn't be handing out links to originals
        # that we don't have (or in the case of thumbs, can't generate).
        # However, someone may have a formerly valid link to a file, so we
        # should do them the favor of giving them a 404.
        try:
            # break apach the url, url-encode it, and put it back together
            urlobj = list(urlparse.urlsplit(reqorig.url))
            # encode the URL but don't encode %s and /s
            urlobj[2] = urllib2.quote(urlobj[2], '%/')
            encodedurl = urlparse.urlunsplit(urlobj)

            # Thumbor never needs URL mangling and it needs a different host
            if self.thumborhost:
                thumbor_reqorig = reqorig.copy()
                thumbor_reqorig.host = self.thumborhost
                thumbor_urlobj = list(urlparse.urlsplit(thumbor_reqorig.url))
                thumbor_urlobj[2] = urllib2.quote(thumbor_urlobj[2], '%/')
                thumbor_encodedurl = urlparse.urlunsplit(thumbor_urlobj)

            # if sitelang, we're supposed to mangle the URL so that
            # http://upload.wikimedia.org/wikipedia/commons/thumb/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
            # changes to http://commons.wikipedia.org/w/thumb_handler.php/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
            if self.backend_url_format == 'sitelang':
                match = re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)', encodedurl)
                if match:
                    proj = match.group('proj')
                    lang = match.group('lang')
                    # and here are all the legacy special cases, imported from thumb_handler.php
                    if(proj == 'wikipedia'):
                        if(lang in ['meta', 'commons', 'internal', 'grants']):
                            proj = 'wikimedia'
                        if(lang in ['mediawiki']):
                            lang = 'www'
                            proj = 'mediawiki'
                    hostname = '%s.%s.org' % (lang, proj)
                    if(proj == 'wikipedia' and lang == 'sources'):
                        #yay special case
                        hostname = 'wikisource.org'
                    # ok, replace the URL with just the part starting with thumb/
                    # take off the first two parts of the path (eg /wikipedia/commons/); make sure the string starts with a /
                    encodedurl = 'http://%s/w/thumb_handler.php/%s' % (hostname, match.group('path'))
                    # add in the X-Original-URI with the swift got (minus the hostname)
                    opener.addheaders.append(('X-Original-URI', list(urlparse.urlsplit(reqorig.url))[2]))
                else:
                    # ASSERT this code should never be hit since only thumbs should call the 404 handler
                    self.logger.warn("non-thumb in 404 handler! encodedurl = %s" % encodedurl)
                    resp = webob.exc.HTTPNotFound('Unexpected error')
                    return resp
            else:
                # log the result of the match here to test and make sure it's sane before enabling the config
                match = re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)', encodedurl)
                if match:
                    proj = match.group('proj')
                    lang = match.group('lang')
                    self.logger.warn("sitelang match has proj %s lang %s encodedurl %s" % (proj, lang, encodedurl))
                else:
                    self.logger.warn("no sitelang match on encodedurl: %s" % encodedurl)

            thumbor_thread = None

            # call thumbor first, otherwise if Mediawiki image scalers return an error,
            # thumbor doesn't get a change to try to generate that thumbnail
            if self.thumborhost:
                if not self.thumbor_wiki_list or '-'.join((proj, lang)) in self.thumbor_wiki_list:
                    # call Thumbor but don't wait for the result
                    thumbor_thread = eventlet.spawn(thumbor_opener.open, thumbor_encodedurl)

            # ok, call the encoded url
            upcopy = opener.open(encodedurl)
            self.collectHttpStatusCodes(original_request_url, thumbor_thread, upcopy.getcode())
        except urllib2.HTTPError, error:
            # copy the urllib2 HTTPError into a webob HTTPError class as-is

            class CopiedHTTPError(webob.exc.HTTPError):
                code = error.code
                title = error.msg

                def html_body(self, environ):
                    return self.detail

                def __init__(self):
                    super(CopiedHTTPError, self).__init__(
                        detail="".join(error.readlines()),
                        headers=error.hdrs.items())

            resp = CopiedHTTPError()
            self.collectHttpStatusCodes(original_request_url, thumbor_thread, resp.code)
            return resp
        except urllib2.URLError, error:
            msg = 'There was a problem while contacting the image scaler: %s' % \
                  error.reason
            resp = webob.exc.HTTPServiceUnavailable(msg)
            self.collectHttpStatusCodes(original_request_url, thumbor_thread, resp.code)
            return resp

        # get the Content-Type.
        uinfo = upcopy.info()
        c_t = uinfo.gettype()<commit_msg>Don’t dual-serve Thumbor traffic anymore<commit_after># Portions Copyright (c) 2010 OpenStack, LLC.
# Everything else Copyright (c) 2011 Wikimedia Foundation, Inc.
# all of it licensed under the Apache Software License, included by reference.

# unit test is in test_rewrite.py. Tests are referenced by numbered comments.

import webob
import webob.exc
import re
import eventlet
from eventlet.green import urllib2
import time
import urlparse
from swift.common.utils import get_logger
from swift.common.wsgi import WSGIContext


class DumbRedirectHandler(urllib2.HTTPRedirectHandler):
    def http_error_301(self, req, fp, code, msg, headers):
        return None

    def http_error_302(self, req, fp, code, msg, headers):
        return None


class _WMFRewriteContext(WSGIContext):
    """
    Rewrite Media Store URLs so that swift knows how to deal with them.
    """

    def __init__(self, rewrite, conf):
        WSGIContext.__init__(self, rewrite.app)
        self.app = rewrite.app
        self.logger = rewrite.logger

        self.account = conf['account'].strip()
        self.thumbhost = conf['thumbhost'].strip()
        self.thumborhost = conf['thumborhost'].strip() if 'thumborhost' in conf else None
        self.thumbor_wiki_list = [item.strip() for item in conf['thumbor_wiki_list'].split(',')] if 'thumbor_wiki_list' in conf else None
        self.user_agent = conf['user_agent'].strip()
        self.bind_port = conf['bind_port'].strip()
        self.shard_container_list = [item.strip() for item in conf['shard_container_list'].split(',')]
        # this parameter controls whether URLs sent to the thumbhost are sent as is (eg. upload/proj/lang/) or with the site/lang
        # converted  and only the path sent back (eg en.wikipedia/thumb).
        self.backend_url_format = conf['backend_url_format'].strip()  # asis, sitelang

    def handle404(self, reqorig, url, container, obj):
        """
        Return a webob.Response which fetches the thumbnail from the thumb
        host and returns it. Note also that the thumb host might write it out
        to Swift so it won't 404 next time.
        """
        original_request_url = reqorig.url
        # go to the thumb media store for unknown files
        reqorig.host = self.thumbhost
        # upload doesn't like our User-agent, otherwise we could call it
        # using urllib2.url()
        proxy_handler = urllib2.ProxyHandler({'http': self.thumbhost})
        redirect_handler = DumbRedirectHandler()

        opener = urllib2.build_opener(redirect_handler, proxy_handler)
        # Thumbor doesn't need (and doesn't like) the proxy
        thumbor_opener = urllib2.build_opener(redirect_handler)

        # Pass on certain headers from the caller squid to the scalers
        opener.addheaders = []
        if reqorig.headers.get('User-Agent') is not None:
            opener.addheaders.append(('User-Agent', reqorig.headers.get('User-Agent')))
        else:
            opener.addheaders.append(('User-Agent', self.user_agent))
        for header_to_pass in ['X-Forwarded-For', 'X-Forwarded-Proto',
                               'Accept', 'Accept-Encoding', 'X-Original-URI']:
            if reqorig.headers.get(header_to_pass) is not None:
                opener.addheaders.append((header_to_pass, reqorig.headers.get(header_to_pass)))

        thumbor_opener.addheaders = opener.addheaders

        self.logger.debug("Addheaders: %r" % thumbor_opener.addheaders)

        # At least in theory, we shouldn't be handing out links to originals
        # that we don't have (or in the case of thumbs, can't generate).
        # However, someone may have a formerly valid link to a file, so we
        # should do them the favor of giving them a 404.
        try:
            # break apach the url, url-encode it, and put it back together
            urlobj = list(urlparse.urlsplit(reqorig.url))
            # encode the URL but don't encode %s and /s
            urlobj[2] = urllib2.quote(urlobj[2], '%/')
            encodedurl = urlparse.urlunsplit(urlobj)

            # Thumbor never needs URL mangling and it needs a different host
            if self.thumborhost:
                thumbor_reqorig = reqorig.copy()
                thumbor_reqorig.host = self.thumborhost
                thumbor_urlobj = list(urlparse.urlsplit(thumbor_reqorig.url))
                thumbor_urlobj[2] = urllib2.quote(thumbor_urlobj[2], '%/')
                thumbor_encodedurl = urlparse.urlunsplit(thumbor_urlobj)

            # if sitelang, we're supposed to mangle the URL so that
            # http://upload.wikimedia.org/wikipedia/commons/thumb/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
            # changes to http://commons.wikipedia.org/w/thumb_handler.php/a/a2/Little_kitten_.jpg/330px-Little_kitten_.jpg
            if self.backend_url_format == 'sitelang':
                match = re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)', encodedurl)
                if match:
                    proj = match.group('proj')
                    lang = match.group('lang')
                    # and here are all the legacy special cases, imported from thumb_handler.php
                    if(proj == 'wikipedia'):
                        if(lang in ['meta', 'commons', 'internal', 'grants']):
                            proj = 'wikimedia'
                        if(lang in ['mediawiki']):
                            lang = 'www'
                            proj = 'mediawiki'
                    hostname = '%s.%s.org' % (lang, proj)
                    if(proj == 'wikipedia' and lang == 'sources'):
                        #yay special case
                        hostname = 'wikisource.org'
                    # ok, replace the URL with just the part starting with thumb/
                    # take off the first two parts of the path (eg /wikipedia/commons/); make sure the string starts with a /
                    encodedurl = 'http://%s/w/thumb_handler.php/%s' % (hostname, match.group('path'))
                    # add in the X-Original-URI with the swift got (minus the hostname)
                    opener.addheaders.append(('X-Original-URI', list(urlparse.urlsplit(reqorig.url))[2]))
                else:
                    # ASSERT this code should never be hit since only thumbs should call the 404 handler
                    self.logger.warn("non-thumb in 404 handler! encodedurl = %s" % encodedurl)
                    resp = webob.exc.HTTPNotFound('Unexpected error')
                    return resp
            else:
                # log the result of the match here to test and make sure it's sane before enabling the config
                match = re.match(r'^http://(?P<host>[^/]+)/(?P<proj>[^-/]+)/(?P<lang>[^/]+)/thumb/(?P<path>.+)', encodedurl)
                if match:
                    proj = match.group('proj')
                    lang = match.group('lang')
                    self.logger.warn("sitelang match has proj %s lang %s encodedurl %s" % (proj, lang, encodedurl))
                else:
                    self.logger.warn("no sitelang match on encodedurl: %s" % encodedurl)

            if self.thumborhost:
                if not self.thumbor_wiki_list or '-'.join((proj, lang)) in self.thumbor_wiki_list:
                    upcopy = thumbor_opener.open(thumbor_encodedurl)
                else:
                    upcopy = opener.open(encodedurl)
            else:
                upcopy = opener.open(encodedurl)

        except urllib2.HTTPError, error:
            # copy the urllib2 HTTPError into a webob HTTPError class as-is

            class CopiedHTTPError(webob.exc.HTTPError):
                code = error.code
                title = error.msg

                def html_body(self, environ):
                    return self.detail

                def __init__(self):
                    super(CopiedHTTPError, self).__init__(
                        detail="".join(error.readlines()),
                        headers=error.hdrs.items())

            resp = CopiedHTTPError()
            return resp
        except urllib2.URLError, error:
            msg = 'There was a problem while contacting the image scaler: %s' % \
                  error.reason
            resp = webob.exc.HTTPServiceUnavailable(msg)
            return resp

        # get the Content-Type.
        uinfo = upcopy.info()
        c_t = uinfo.gettype()
{'fraction_non_alphanumeric': 0.1006729920564872, 'fraction_numerical': 0.006123124448367167, 'mean_word_length': 2.080020387359837, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3103018', 'n_tokens_mistral': 5271, 'n_tokens_neox': 5038, 'n_words': 1436}