<commit_before>from __future__ import with_statement

import hashlib
import logging
import mimetypes
import os
import posixpath
import sys

from genshi import HTML
from genshi.builder import tag
from genshi.template import TemplateLoader
from genshi.filters import Transformer, HTMLFormFiller
from werkzeug import Local, LocalManager, SharedDataMiddleware, Request, \
                     Response, ClosingIterator, DebuggedApplication, \
                     serving, redirect
from werkzeug.exceptions import HTTPException, BadRequest, NotFound
from werkzeug.routing import Map, Rule
from werkzeug.utils import cached_property
from werkzeug.contrib.sessions import Session, FilesystemSessionStore, generate_key
import genshi
import simplejson

from flam.util import Signal
from flam import validate


__all__ = [
    'expose', 'run_server', 'static_resource', 'json', 'Request', 'Response',
    'application', 'local', 'html', 'request', 'href', 'redirect', 'static',
    'flash', 'INFO', 'WARNING', 'ERROR','context_setup', 'request_setup',
    'request_teardown', 'HTML', 'tag', 'user', 'session', 'process_form',
    ]


local = Local()
local_manager = LocalManager([local])
application = local('application')
flash_message = local('flash_message')
request = local('request')
session = local('session')
user = local('session')
# URL routing rules
url_map = Map([Rule('/static/<file>', endpoint='static', build_only=True)])
# URL routing endpoint to callback mapping.
view_map = {}
# Global template loader object
template_loader = None
# Session store
session_store = None


# flash() levels
INFO = 'info'
WARNING = 'warning'
ERROR = 'error'


class Callback(Signal):
    """Signal decorator."""
    def __init__(self, help, limit=None):
        super(Callback, self).__init__(limit=limit)
        self.__doc__ = help

    def dispatch(self, *args, **kwargs):
        return super(Callback, self).__call__(*args, **kwargs)

    def __call__(self, *args, **kwargs):
        return super(Callback, self).connect(*args, **kwargs)


context_setup = Callback('Register a template render context setup callback.')
request_setup = Callback('Register a request setup function.')
request_teardown = Callback('Register a request teardown function.')


class Request(Request):
    """Request object with some useful extra methods."""

    @cached_property
    def form_token(self):
        if 'form_token' in self.cookies:
            return self.cookies['form_token']
        else:
            return generate_key()

    @cached_property
    def json(self):
        return simplejson.loads(self.data)


class Application(object):
    """Core WSGI application."""

    def __init__(self, cookie_name=None, debug=False, static_root=None, static_map=None):
        local.application = self
        self.cookie_name = cookie_name or os.path.basename(sys.argv[0])
        self.debug = debug
        self.static_root = static_root
        self.static_map = static_map

    def __call__(self, environ, start_response):
        request = Request(environ)
        local.application = self
        local.request = request
        local.url_adapter = adapter = url_map.bind_to_environ(environ)
        local.flash_message = {}
        local.session = self._create_session()
        local.user = None

        request_setup.dispatch()

        try:
            # CSRF protection concept borrowed from Trac.
            session_form_token = request.form_token
            form_token = request.form.get('__FORM_TOKEN')
            if request.method == 'POST' and form_token != session_form_token:
                logging.warning('Invalid form token %s != %s from %r',
                                form_token, session_form_token, request)
                raise BadRequest('Invalid form token, do you have cookies enabled?')

            endpoint, values = adapter.match()
            handler = view_map[endpoint]
            if 'flash' in session:
                flash_message.update(session['flash'])
                del session['flash']
            response = handler(**values)
            if isinstance(response, genshi.Stream):
                token = tag.input(type='hidden', name='__FORM_TOKEN',
                                  value=session_form_token)
                response = response | Transformer('//form').prepend(token)
                if self.static_root:
                    response = self._remap_static(response)
                response = Response(response.render('xhtml'), content_type='text/html')
                response.set_cookie('form_token', session_form_token, httponly=True)
            elif flash_message:
                session['flash'] = dict(flash_message)

            if session.should_save:
                session_store.save(session)
                response.set_cookie(
                    self.cookie_name, session.sid,
                    httponly=True,
                    )
        except HTTPException, e:
            response = e
        return ClosingIterator(response(environ, start_response),
                               [request_teardown.dispatch])

    def _create_session(self):
        sid = request.cookies.get(self.cookie_name)
        if sid is None:
            return session_store.new()
        else:
            return session_store.get(sid)

    def _remap_static(self, stream, prefix='/static/'):
        """Remap links to static content to the correct URL root."""
        def map_static(name, event):
            attrs = event[1][1]
            name = attrs.get(name)[len(prefix):]
            if self.static_map:
                name = self.static_map.get(name, name)
            return static(name)
        return stream | Transformer('//*[matches(@src, "^%s")]' % prefix).attr('src', map_static) | \
            Transformer('//*[matches(@href, "^%s")]' % prefix).attr('href', map_static)


def flash(message, type=INFO):
    """Flash a message to the user on next request."""
    flash_message['message'] = message
    flash_message['type'] = type


@context_setup
def default_context_setup(context):
    """Populate the default template render context."""
    context['href'] = href
    context['static'] = static
    context['session'] = session
    context['flash'] = flash_message
    context['debug'] = application.debug
    context['Markup'] = genshi.Markup


def expose(path=None, **kw):
    """Expose a function as a routing endpoint.

    If arguments are omitted, the wrapped function name will be used as the
    rule name and routing path.
    """
    def decorate(function):
        endpoint = function.__name__
        kw.setdefault('endpoint', endpoint)
        view_map[endpoint] = function
        rule = Rule('/' + function.__name__ if path is None else path, **kw)
        url_map.add(rule)
        function._routing_rule = rule
        return function

    if callable(path):
        # decorate() assumes "path" is a string or None, we set it to the
        # latter to force auto-pathing.
        function = path
        path = None
        return decorate(function)

    return decorate


class Href(object):
    """A convenience object for referring to endpoints."""
    def __getattr__(self, endpoint):
        endpoint = [endpoint]
        def wrapper(_external=False, **values):
            if callable(endpoint[0]):
                endpoint[0] = endpoint[0].__name__
            return local.url_adapter.build(endpoint[0], values, force_external=_external)
        return wrapper

    __getitem__ = __getattr__


href = Href()


def static(filename):
    """Convenience function for referring to static content."""
    return href.static(file=filename)


def static_resource(filename):
    """Serve a static resource."""
    mime_type, encoding = mimetypes.guess_type(filename)
    try:
      fd = open(filename)
    except EnvironmentError:
      raise NotFound()
    try:
      return Response(fd.read(), content_type=mime_type)
    finally:
        fd.close()


def json(data):
    """Convert a fundamental Python object to a JSON response."""
    return Response(simplejson.dumps(data), content_type='application/json')


def html(template, **data):
    """Render a HTML template."""
    tmpl = template_loader.load(template)
    context = {}
    context_setup.dispatch(context)
    context.update(data)
    stream = tmpl.generate(**context)
    return stream


def load_static_map(mapping_file):
    """Load a mapping of real filenames to pre-processed hashed filenames.

    The mapping file should consist of lines in the format:
        <hash>.<ext> <filename>.<ext>
    eg.
        243045720e30559c1a77e6ef1585a76c.js jquery.corners.min.js
    """
    map = {}
    with open(mapping_file) as fd:
        for line in fd:
            v, k = line.strip().split(None, 1)
            map[k] = v
    return map


def run_server(host='localhost', port=0xdead, static_path=None, debug=False,
               log_level=logging.WARNING, template_paths=None,
               cookie_name=None, static_map=None):
    """Start a new standalone application server."""
    global template_loader, session_store
    session_store = FilesystemSessionStore()
    template_loader = TemplateLoader(template_paths or ['templates'],
                                     auto_reload=debug)

    if isinstance(static_map, basestring):
        static_map = load_static_map(static_map)
    application = Application(cookie_name=cookie_name, debug=debug,
                              static_root='/static', static_map=static_map)
    if debug:
        application = DebuggedApplication(application, evalex=True)
        log_level = logging.DEBUG
    logging.basicConfig(level=log_level)
    logging.getLogger('werkzeug').setLevel(log_level)
    logging.getLogger().setLevel(log_level)
    if not static_path:
        static_path = os.path.join(os.getcwd(), 'static')
    application = SharedDataMiddleware(application, {'/static': static_path})
    serving.run_simple(host, port, application, use_reloader=debug)


def process_form(template, validator, **context):
    """Perform "typical" processing of a template form.

    This means checking that a form has been POSTed and is valid.

    :param template: A Genshi stream, or template filename as a string.
    :param validator: flam.validate.Form object.
    :param context: Template context parameters.
    :returns: A tuple of (valid, response).
    """
    if isinstance(template, basestring):
        template = html(template, **context)
    form = request.form

    if not form:
        return False, template

    validation_context = validator.validate(form)
    if validation_context.errors:
        return False, template | HTMLFormFiller(data=form) | validation_context.inject_errors()

    return True, template
<commit_msg>Fixed a small but annoying bug with the "user" thread local variable.<commit_after>from __future__ import with_statement

import hashlib
import logging
import mimetypes
import os
import posixpath
import sys

from genshi import HTML
from genshi.builder import tag
from genshi.template import TemplateLoader
from genshi.filters import Transformer, HTMLFormFiller
from werkzeug import Local, LocalManager, SharedDataMiddleware, Request, \
                     Response, ClosingIterator, DebuggedApplication, \
                     serving, redirect
from werkzeug.exceptions import HTTPException, BadRequest, NotFound
from werkzeug.routing import Map, Rule
from werkzeug.utils import cached_property
from werkzeug.contrib.sessions import Session, FilesystemSessionStore, generate_key
import genshi
import simplejson

from flam.util import Signal
from flam import validate


__all__ = [
    'expose', 'run_server', 'static_resource', 'json', 'Request', 'Response',
    'application', 'local', 'html', 'request', 'href', 'redirect', 'static',
    'flash', 'INFO', 'WARNING', 'ERROR','context_setup', 'request_setup',
    'request_teardown', 'HTML', 'tag', 'user', 'session', 'process_form',
    ]


local = Local()
local_manager = LocalManager([local])
application = local('application')
flash_message = local('flash_message')
request = local('request')
session = local('session')
user = local('user')
# URL routing rules
url_map = Map([Rule('/static/<file>', endpoint='static', build_only=True)])
# URL routing endpoint to callback mapping.
view_map = {}
# Global template loader object
template_loader = None
# Session store
session_store = None


# flash() levels
INFO = 'info'
WARNING = 'warning'
ERROR = 'error'


class Callback(Signal):
    """Signal decorator."""
    def __init__(self, help, limit=None):
        super(Callback, self).__init__(limit=limit)
        self.__doc__ = help

    def dispatch(self, *args, **kwargs):
        return super(Callback, self).__call__(*args, **kwargs)

    def __call__(self, *args, **kwargs):
        return super(Callback, self).connect(*args, **kwargs)


context_setup = Callback('Register a template render context setup callback.')
request_setup = Callback('Register a request setup function.')
request_teardown = Callback('Register a request teardown function.')


class Request(Request):
    """Request object with some useful extra methods."""

    @cached_property
    def form_token(self):
        if 'form_token' in self.cookies:
            return self.cookies['form_token']
        else:
            return generate_key()

    @cached_property
    def json(self):
        return simplejson.loads(self.data)


class Application(object):
    """Core WSGI application."""

    def __init__(self, cookie_name=None, debug=False, static_root=None, static_map=None):
        local.application = self
        self.cookie_name = cookie_name or os.path.basename(sys.argv[0])
        self.debug = debug
        self.static_root = static_root
        self.static_map = static_map

    def __call__(self, environ, start_response):
        request = Request(environ)
        local.application = self
        local.request = request
        local.url_adapter = adapter = url_map.bind_to_environ(environ)
        local.flash_message = {}
        local.session = self._create_session()
        local.user = None

        request_setup.dispatch()

        try:
            # CSRF protection concept borrowed from Trac.
            session_form_token = request.form_token
            form_token = request.form.get('__FORM_TOKEN')
            if request.method == 'POST' and form_token != session_form_token:
                logging.warning('Invalid form token %s != %s from %r',
                                form_token, session_form_token, request)
                raise BadRequest('Invalid form token, do you have cookies enabled?')

            endpoint, values = adapter.match()
            handler = view_map[endpoint]
            if 'flash' in session:
                flash_message.update(session['flash'])
                del session['flash']
            response = handler(**values)
            if isinstance(response, genshi.Stream):
                token = tag.input(type='hidden', name='__FORM_TOKEN',
                                  value=session_form_token)
                response = response | Transformer('//form').prepend(token)
                if self.static_root:
                    response = self._remap_static(response)
                response = Response(response.render('xhtml'), content_type='text/html')
                response.set_cookie('form_token', session_form_token, httponly=True)
            elif flash_message:
                session['flash'] = dict(flash_message)

            if session.should_save:
                session_store.save(session)
                response.set_cookie(
                    self.cookie_name, session.sid,
                    httponly=True,
                    )
        except HTTPException, e:
            response = e
        return ClosingIterator(response(environ, start_response),
                               [request_teardown.dispatch])

    def _create_session(self):
        sid = request.cookies.get(self.cookie_name)
        if sid is None:
            return session_store.new()
        else:
            return session_store.get(sid)

    def _remap_static(self, stream, prefix='/static/'):
        """Remap links to static content to the correct URL root."""
        def map_static(name, event):
            attrs = event[1][1]
            name = attrs.get(name)[len(prefix):]
            if self.static_map:
                name = self.static_map.get(name, name)
            return static(name)
        return stream | Transformer('//*[matches(@src, "^%s")]' % prefix).attr('src', map_static) | \
            Transformer('//*[matches(@href, "^%s")]' % prefix).attr('href', map_static)


def flash(message, type=INFO):
    """Flash a message to the user on next request."""
    flash_message['message'] = message
    flash_message['type'] = type


@context_setup
def default_context_setup(context):
    """Populate the default template render context."""
    context['href'] = href
    context['static'] = static
    context['session'] = session
    context['flash'] = flash_message
    context['debug'] = application.debug
    context['Markup'] = genshi.Markup


def expose(path=None, **kw):
    """Expose a function as a routing endpoint.

    If arguments are omitted, the wrapped function name will be used as the
    rule name and routing path.
    """
    def decorate(function):
        endpoint = function.__name__
        kw.setdefault('endpoint', endpoint)
        view_map[endpoint] = function
        rule = Rule('/' + function.__name__ if path is None else path, **kw)
        url_map.add(rule)
        function._routing_rule = rule
        return function

    if callable(path):
        # decorate() assumes "path" is a string or None, we set it to the
        # latter to force auto-pathing.
        function = path
        path = None
        return decorate(function)

    return decorate


class Href(object):
    """A convenience object for referring to endpoints."""
    def __getattr__(self, endpoint):
        endpoint = [endpoint]
        def wrapper(_external=False, **values):
            if callable(endpoint[0]):
                endpoint[0] = endpoint[0].__name__
            return local.url_adapter.build(endpoint[0], values, force_external=_external)
        return wrapper

    __getitem__ = __getattr__


href = Href()


def static(filename):
    """Convenience function for referring to static content."""
    return href.static(file=filename)


def static_resource(filename):
    """Serve a static resource."""
    mime_type, encoding = mimetypes.guess_type(filename)
    try:
      fd = open(filename)
    except EnvironmentError:
      raise NotFound()
    try:
      return Response(fd.read(), content_type=mime_type)
    finally:
        fd.close()


def json(data):
    """Convert a fundamental Python object to a JSON response."""
    return Response(simplejson.dumps(data), content_type='application/json')


def html(template, **data):
    """Render a HTML template."""
    tmpl = template_loader.load(template)
    context = {}
    context_setup.dispatch(context)
    context.update(data)
    stream = tmpl.generate(**context)
    return stream


def load_static_map(mapping_file):
    """Load a mapping of real filenames to pre-processed hashed filenames.

    The mapping file should consist of lines in the format:
        <hash>.<ext> <filename>.<ext>
    eg.
        243045720e30559c1a77e6ef1585a76c.js jquery.corners.min.js
    """
    map = {}
    with open(mapping_file) as fd:
        for line in fd:
            v, k = line.strip().split(None, 1)
            map[k] = v
    return map


def run_server(host='localhost', port=0xdead, static_path=None, debug=False,
               log_level=logging.WARNING, template_paths=None,
               cookie_name=None, static_map=None):
    """Start a new standalone application server."""
    global template_loader, session_store
    session_store = FilesystemSessionStore()
    template_loader = TemplateLoader(template_paths or ['templates'],
                                     auto_reload=debug)

    if isinstance(static_map, basestring):
        static_map = load_static_map(static_map)
    application = Application(cookie_name=cookie_name, debug=debug,
                              static_root='/static', static_map=static_map)
    if debug:
        application = DebuggedApplication(application, evalex=True)
        log_level = logging.DEBUG
    logging.basicConfig(level=log_level)
    logging.getLogger('werkzeug').setLevel(log_level)
    logging.getLogger().setLevel(log_level)
    if not static_path:
        static_path = os.path.join(os.getcwd(), 'static')
    application = SharedDataMiddleware(application, {'/static': static_path})
    serving.run_simple(host, port, application, use_reloader=debug)


def process_form(template, validator, **context):
    """Perform "typical" processing of a template form.

    This means checking that a form has been POSTed and is valid.

    :param template: A Genshi stream, or template filename as a string.
    :param validator: flam.validate.Form object.
    :param context: Template context parameters.
    :returns: A tuple of (valid, response).
    """
    if isinstance(template, basestring):
        template = html(template, **context)
    form = request.form

    if not form:
        return False, template

    validation_context = validator.validate(form)
    if validation_context.errors:
        return False, template | HTMLFormFiller(data=form) | validation_context.inject_errors()

    return True, template

{'fraction_non_alphanumeric': 0.1053486106601104, 'fraction_numerical': 0.0030616505079556523, 'mean_word_length': 2.8627486113599714, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4464104', 'n_tokens_mistral': 6006, 'n_tokens_neox': 5841, 'n_words': 1597}