<commit_before>from __future__ import division

from cStringIO import StringIO
from subprocess import call
from urllib import urlencode, quote as urlquote
from urllib2 import urlopen
from urlparse import urlparse
import base64
import datetime
import errno
import hashlib
import logging
import math
import os
import re
import struct
import sys

from PIL import Image as image
from flask import request, current_app, send_file, abort
from itsdangerous import Signer, constant_time_compare


log = logging.getLogger(__name__)


def encode_int(value):
    return base64.urlsafe_b64encode(struct.pack('>I', int(value))).rstrip('=').lstrip('A')


def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise


# We must whitelist schemes which are permitted, otherwise craziness (such as
# allowing access to the filesystem) may ensue.
ALLOWED_SCHEMES = set(('http', 'https', 'ftp'))


class Images(object):
    
    MODE_FIT = 'fit'
    MODE_CROP = 'crop'
    MODE_PAD = 'pad'
    MODE_RESHAPE = 'reshape'
    MODES = (MODE_FIT, MODE_CROP, MODE_PAD, MODE_RESHAPE)
    
    def __init__(self, app=None):
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['images'] = self

        app.config.setdefault('IMAGES_URL', '/imgsizer') # This is historical.
        app.config.setdefault('IMAGES_NAME', 'images')
        app.config.setdefault('IMAGES_PATH', ['static'])
        app.config.setdefault('IMAGES_CACHE', '/tmp/flask-images')
        app.config.setdefault('IMAGES_MAX_AGE', 3600)

        app.add_url_rule(app.config['IMAGES_URL'] + '/<path:path>', app.config['IMAGES_NAME'], self.handle_request)
        app.context_processor(self._context_processor)
        app.url_build_error_handlers.append(self.build_error_handler)

    def build_error_handler(self, error, endpoint, values):

        # See if we were asked for "images" or "images.<mode>".
        m = re.match(r'^%s(?:\.(%s))?$' % (
            re.escape(current_app.config['IMAGES_NAME']),
            '|'.join(re.escape(mode) for mode in self.MODES)
        ), endpoint)
        if m:
            
            filename = values.pop('filename')
            mode = m.group(1)

            # This is slightly awkward, but I want to trigger the built-in
            # TypeError if you use the "images.<mode>" method AND provide
            # a "mode" kwarg.
            if mode:
                return self.build_url(filename, mode=mode, **values)
            return self.build_url(filename, **values)

        return None

    def _context_processor(self):
        return dict(
            resized_img_src=resized_img_src,
        )

    def build_url(self, local_path, **kwargs):

        # Make the path relative.
        local_path = local_path.strip('/')

        # We complain when we see non-normalized paths, as it is a good
        # indicator that unsanitized data may be getting through.
        # Mutating the scheme syntax to match is a little gross, but it works
        # for today.
        norm_path = os.path.normpath(local_path)
        if local_path.replace('://', ':/') != norm_path or norm_path.startswith('../'):
            raise ValueError('path is not normalized')

        for key in 'background mode width height quality format padding'.split():
            if key in kwargs:
                kwargs[key[0]] = kwargs.pop(key)<commit_msg>Use add_template_global instead of context_processor<commit_after>from __future__ import division

from cStringIO import StringIO
from subprocess import call
from urllib import urlencode, quote as urlquote
from urllib2 import urlopen
from urlparse import urlparse
import base64
import datetime
import errno
import hashlib
import logging
import math
import os
import re
import struct
import sys

from PIL import Image as image
from flask import request, current_app, send_file, abort
from itsdangerous import Signer, constant_time_compare


log = logging.getLogger(__name__)


def encode_int(value):
    return base64.urlsafe_b64encode(struct.pack('>I', int(value))).rstrip('=').lstrip('A')


def makedirs(path):
    try:
        os.makedirs(path)
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise


# We must whitelist schemes which are permitted, otherwise craziness (such as
# allowing access to the filesystem) may ensue.
ALLOWED_SCHEMES = set(('http', 'https', 'ftp'))


class Images(object):
    
    MODE_FIT = 'fit'
    MODE_CROP = 'crop'
    MODE_PAD = 'pad'
    MODE_RESHAPE = 'reshape'
    MODES = (MODE_FIT, MODE_CROP, MODE_PAD, MODE_RESHAPE)
    
    def __init__(self, app=None):
        if app is not None:
            self.init_app(app)

    def init_app(self, app):
        if not hasattr(app, 'extensions'):
            app.extensions = {}
        app.extensions['images'] = self

        app.config.setdefault('IMAGES_URL', '/imgsizer') # This is historical.
        app.config.setdefault('IMAGES_NAME', 'images')
        app.config.setdefault('IMAGES_PATH', ['static'])
        app.config.setdefault('IMAGES_CACHE', '/tmp/flask-images')
        app.config.setdefault('IMAGES_MAX_AGE', 3600)

        app.add_url_rule(app.config['IMAGES_URL'] + '/<path:path>', app.config['IMAGES_NAME'], self.handle_request)
        app.url_build_error_handlers.append(self.build_error_handler)

        app.add_template_global(resized_img_src)

    def build_error_handler(self, error, endpoint, values):

        # See if we were asked for "images" or "images.<mode>".
        m = re.match(r'^%s(?:\.(%s))?$' % (
            re.escape(current_app.config['IMAGES_NAME']),
            '|'.join(re.escape(mode) for mode in self.MODES)
        ), endpoint)
        if m:
            
            filename = values.pop('filename')
            mode = m.group(1)

            # This is slightly awkward, but I want to trigger the built-in
            # TypeError if you use the "images.<mode>" method AND provide
            # a "mode" kwarg.
            if mode:
                return self.build_url(filename, mode=mode, **values)
            return self.build_url(filename, **values)

        return None

    def build_url(self, local_path, **kwargs):

        # Make the path relative.
        local_path = local_path.strip('/')

        # We complain when we see non-normalized paths, as it is a good
        # indicator that unsanitized data may be getting through.
        # Mutating the scheme syntax to match is a little gross, but it works
        # for today.
        norm_path = os.path.normpath(local_path)
        if local_path.replace('://', ':/') != norm_path or norm_path.startswith('../'):
            raise ValueError('path is not normalized')

        for key in 'background mode width height quality format padding'.split():
            if key in kwargs:
                kwargs[key[0]] = kwargs.pop(key)
{'fraction_non_alphanumeric': 0.10746525290156182, 'fraction_numerical': 0.003725462100587477, 'mean_word_length': 2.740621650589496, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7270874', 'n_tokens_mistral': 2162, 'n_tokens_neox': 2084, 'n_words': 598}