<commit_before>from .json_script_escape import json_script_escape
from pkg_resources import resource_filename
from pyramid.events import (
    NewRequest,
    subscriber,
)
from pyramid.decorator import reify
from pyramid.httpexceptions import (
    HTTPBadRequest,
    HTTPServerError,
    HTTPMovedPermanently,
)
from pyramid.security import authenticated_userid
from pyramid.threadlocal import (
    get_current_request,
    manager,
)
import atexit
import json
import logging
import os
import pyramid.renderers
import subprocess
import threading
import time
import uuid
import weakref


log = logging.getLogger(__name__)


def includeme(config):
    config.add_renderer(None, PageOrJSON)
    config.add_renderer('null_renderer', NullRenderer)
    config.scan(__name__)


class JSON(pyramid.renderers.JSON):
    '''Provide easier access to the configured serializer
    '''
    def dumps(self, value):
        request = get_current_request()
        default = json_renderer._make_default(request)
        return self.serializer(value, default=default, **self.kw)


json_renderer = JSON()


def uuid_adapter(obj, request):
    return str(obj)


json_renderer.add_adapter(uuid.UUID, uuid_adapter)


class NullRenderer:
    '''Sets result value directly as response.
    '''
    def __init__(self, info):
        pass

    def __call__(self, value, system):
        request = system.get('request')
        if request is None:
            return value
        request.response = value
        return None


class RenderingError(HTTPServerError):
    title = 'Server Rendering Error'
    explanation = 'The server erred while rendering the page.'


class RetryRender(Exception):
    def __init__(self, phase):
        self.phase = phase


def cleanup(plist):
    for process in plist:
        if process.stdin is not None:
            process.stdin.close()
        if process.stdout is not None:
            process.stdout.close()
        if process.stderr is not None:
            process.stderr.close()
        if process.poll() is None:
            process.terminate()

    # sum((0.01 * (2 ** n - 1)) for n in range(6)) -> 0.57
    for n in xrange(6):
        time.sleep(0.01 * (2 ** n - 1))
        if all(process.poll() is not None for process in plist):
            break

    for process in plist:
        if process.poll() is None:
            process.kill()


# Hold a weakreference to each subprocess for cleanup during shutdown
renderer_processes = weakref.WeakSet()


@atexit.register
def cleanup_processes():
    cleanup(list(renderer_processes))


class PageWorker(threading.local):
    """ We only want one of these per thread
    """
    process_args = [
        'node',
        resource_filename(__name__, 'static/build/renderer.js'),
    ]

    @reify
    def process(self):
        """ defer creation as __init__ also called in management thread
        """
        node_env = os.environ.copy()
        node_env['NODE_PATH']= ''
        process = subprocess.Popen(
            self.process_args, close_fds=True,
            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            env=node_env,
        )
        renderer_processes.add(process)
        return process

    def render(self, value, system):
        request = system['request']
        request.response.content_type = 'text/html'
        data = '{"href":%s,"context":%s}\0' % (json.dumps(request.url), value)

        for attempt in range(2):
            process = self.process
            try:
                start = int(time.time() * 1e6)
                process.stdin.write(data)
                header = process.stdout.readline()
                if not header:
                    raise RetryRender('header')
                try:
                    result_type, content_length = header.split(' ', 1)
                except ValueError:
                    raise Exception('Renderer header malformed: %r' % header)
                content_length = int(content_length)
                output = []
                pos = 0

                while pos < content_length:
                    out = process.stdout.read(content_length - pos)
                    if not out:
                        raise RetryRender('body')
                    pos += len(out)
                    output.append(out)

                end = int(time.time() * 1e6)

            except Exception as e:
                del self.process
                cleanup([process])
                if not isinstance(e, RetryRender):
                    raise
                log.error(
                    'Renderer closed pipe (phase: %s, attempt: %d)',
                    e.phase, attempt,
                )
            else:
                break
        else:
            raise Exception('Renderer closed pipe unexpectedly')

        stats = request._stats
        stats['render_count'] = stats.get('render_count', 0) + 1
        duration = end - start
        stats['render_time'] = stats.get('render_time', 0) + duration

        result = ''.join(output)
        if result_type == 'RESULT':
            return result
        else:
            raise RenderingError(result)<commit_msg>Make the javascript renderer reload each request when pyramid.debug_templates=true.<commit_after>from .json_script_escape import json_script_escape
from pkg_resources import resource_filename
from pyramid.events import (
    NewRequest,
    subscriber,
)
from pyramid.decorator import reify
from pyramid.httpexceptions import (
    HTTPBadRequest,
    HTTPServerError,
    HTTPMovedPermanently,
)
from pyramid.security import authenticated_userid
from pyramid.threadlocal import (
    get_current_request,
    manager,
)
import atexit
import json
import logging
import os
import pyramid.renderers
import subprocess
import threading
import time
import uuid
import weakref


log = logging.getLogger(__name__)


def includeme(config):
    config.add_renderer(None, PageOrJSON)
    config.add_renderer('null_renderer', NullRenderer)
    config.scan(__name__)


class JSON(pyramid.renderers.JSON):
    '''Provide easier access to the configured serializer
    '''
    def dumps(self, value):
        request = get_current_request()
        default = json_renderer._make_default(request)
        return self.serializer(value, default=default, **self.kw)


json_renderer = JSON()


def uuid_adapter(obj, request):
    return str(obj)


json_renderer.add_adapter(uuid.UUID, uuid_adapter)


class NullRenderer:
    '''Sets result value directly as response.
    '''
    def __init__(self, info):
        pass

    def __call__(self, value, system):
        request = system.get('request')
        if request is None:
            return value
        request.response = value
        return None


class RenderingError(HTTPServerError):
    title = 'Server Rendering Error'
    explanation = 'The server erred while rendering the page.'


class RetryRender(Exception):
    def __init__(self, phase):
        self.phase = phase


def cleanup(plist):
    for process in plist:
        if process.stdin is not None:
            process.stdin.close()
        if process.stdout is not None:
            process.stdout.close()
        if process.stderr is not None:
            process.stderr.close()
        if process.poll() is None:
            process.terminate()

    # sum((0.01 * (2 ** n - 1)) for n in range(6)) -> 0.57
    for n in xrange(6):
        time.sleep(0.01 * (2 ** n - 1))
        if all(process.poll() is not None for process in plist):
            break

    for process in plist:
        if process.poll() is None:
            process.kill()


# Hold a weakreference to each subprocess for cleanup during shutdown
renderer_processes = weakref.WeakSet()


@atexit.register
def cleanup_processes():
    cleanup(list(renderer_processes))


class PageWorker(threading.local):
    """ We only want one of these per thread
    """
    process_args = [
        'node',
        resource_filename(__name__, 'static/build/renderer.js'),
    ]

    @reify
    def process(self):
        """ defer creation as __init__ also called in management thread
        """
        node_env = os.environ.copy()
        node_env['NODE_PATH']= ''
        process = subprocess.Popen(
            self.process_args, close_fds=True,
            stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE,
            env=node_env,
        )
        renderer_processes.add(process)
        return process

    def render(self, value, system):
        request = system['request']
        request.response.content_type = 'text/html'
        data = '{"href":%s,"context":%s}\0' % (json.dumps(request.url), value)

        for attempt in range(2):
            process = self.process
            try:
                start = int(time.time() * 1e6)
                process.stdin.write(data)
                header = process.stdout.readline()
                if not header:
                    raise RetryRender('header')
                try:
                    result_type, content_length = header.split(' ', 1)
                except ValueError:
                    raise Exception('Renderer header malformed: %r' % header)
                content_length = int(content_length)
                output = []
                pos = 0

                while pos < content_length:
                    out = process.stdout.read(content_length - pos)
                    if not out:
                        raise RetryRender('body')
                    pos += len(out)
                    output.append(out)

                end = int(time.time() * 1e6)

            except Exception as e:
                del self.process
                cleanup([process])
                if not isinstance(e, RetryRender):
                    raise
                log.error(
                    'Renderer closed pipe (phase: %s, attempt: %d)',
                    e.phase, attempt,
                )
            else:
                break
        else:
            raise Exception('Renderer closed pipe unexpectedly')

        stats = request._stats
        stats['render_count'] = stats.get('render_count', 0) + 1
        duration = end - start
        stats['render_time'] = stats.get('render_time', 0) + duration

        if request.registry.settings['pyramid.debug_templates']:
            del self.process
            cleanup([process])

        result = ''.join(output)
        if result_type == 'RESULT':
            return result
        else:
            raise RenderingError(result)
{'fraction_non_alphanumeric': 0.09281095473564092, 'fraction_numerical': 0.00494484594903005, 'mean_word_length': 2.040474125469789, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3481957', 'n_tokens_mistral': 2956, 'n_tokens_neox': 2806, 'n_words': 768}