<commit_before>import os

from StringIO import StringIO

from twisted.trial import unittest

from klein import Klein

from klein.interfaces import IKleinRequest
from klein.resource import KleinResource, ensure_utf8_bytes

from twisted.internet.defer import succeed, Deferred, fail, CancelledError
from twisted.internet.error import ConnectionLost
from twisted.internet.task import Clock
from twisted.web import server
from twisted.web.static import File
from twisted.web.resource import Resource
from twisted.web.template import Element, XMLString, renderer
from twisted.web.test.test_web import DummyChannel
from twisted.web.http_headers import Headers

from werkzeug.exceptions import NotFound

from mock import Mock, call


def requestMock(path, method="GET", host="localhost", port=8080, isSecure=False,
                body=None, headers=None):
    if not headers:
        headers = {}

    if not body:
        body = ''

    request = server.Request(DummyChannel(), False)
    request.site = Mock(server.Site)
    request.gotLength(len(body))
    request.content = StringIO()
    request.content.write(body)
    request.content.seek(0)
    request.requestHeaders = Headers(headers)
    request.setHost(host, port, isSecure)
    request.uri = path
    request.prepath = []
    request.postpath = path.split('/')[1:]
    request.method = method
    request.clientproto = 'HTTP/1.1'

    request.setHeader = Mock(wraps=request.setHeader)
    request.setResponseCode = Mock(wraps=request.setResponseCode)

    request.testClock = Clock()
    request._written = StringIO()

    request._finishCalled = 0
    request._writeCalled = 0

    def produce():
        while request.producer:
            request.producer.resumeProducing()

    def registerProducer(producer, streaming):
        request.producer = producer
        request.producer.resumeProducing()
        if not streaming:
            request.testClock.callLater(0.0, produce)
            request.testClock.advance(0)

    def unregisterProducer():
        request.producer = None

    def finish():
        request._finishCalled += 1

        if not request.startedWriting:
            request.write('')

        if not request.finished:
            request.finished = True
            request._cleanup()

    def write(data):
        request._writeCalled += 1
        request.startedWriting = True

        if not request.finished:
            request._written.write(data)
        else:
            raise RuntimeError('Request.write called on a request after '
                'Request.finish was called.')

    def assertWritten(data):
        responseData = request._written.getvalue()
        if not responseData == data:
            raise AssertionError("Same data was not written!\ngot: %s \nexpected: %s" % (responseData, data))
        else:
            return True

    def assertWrittenOnceWith(data):
        if request.assertWritten(data) and request._writeCalled == 1:
            return True
        else:
            raise AssertionError("request.write was called %s times, expected 1" % (request._writeCalled,))

    def assertFinishedOnce():
        if request._finishCalled == 1:
            return True
        else:
            raise AssertionError("request.finish was called %s times, expected 1" % (request._finishCalled,))

    request.finish = finish
    request.write = write
    request.assertWritten = assertWritten
    request.assertWrittenOnceWith = assertWrittenOnceWith
    request.assertFinishedOnce = assertFinishedOnce

    request.registerProducer = registerProducer
    request.unregisterProducer = unregisterProducer

    request.processingFailed = Mock(wraps=request.processingFailed)

    return request


def _render(resource, request):
    result = resource.render(request)

    if isinstance(result, str):
        request.write(result)
        request.finish()
        return succeed(None)
    elif result is server.NOT_DONE_YET:
        if request.finished:
            return succeed(None)
        else:
            return request.notifyFinish()
    else:
        raise ValueError("Unexpected return value: %r" % (result,))

<commit_msg>80 lines<commit_after>import os

from StringIO import StringIO

from twisted.trial import unittest

from klein import Klein

from klein.interfaces import IKleinRequest
from klein.resource import KleinResource, ensure_utf8_bytes

from twisted.internet.defer import succeed, Deferred, fail, CancelledError
from twisted.internet.error import ConnectionLost
from twisted.internet.task import Clock
from twisted.web import server
from twisted.web.static import File
from twisted.web.resource import Resource
from twisted.web.template import Element, XMLString, renderer
from twisted.web.test.test_web import DummyChannel
from twisted.web.http_headers import Headers

from werkzeug.exceptions import NotFound

from mock import Mock, call


def requestMock(path, method="GET", host="localhost", port=8080, isSecure=False,
                body=None, headers=None):
    if not headers:
        headers = {}

    if not body:
        body = ''

    request = server.Request(DummyChannel(), False)
    request.site = Mock(server.Site)
    request.gotLength(len(body))
    request.content = StringIO()
    request.content.write(body)
    request.content.seek(0)
    request.requestHeaders = Headers(headers)
    request.setHost(host, port, isSecure)
    request.uri = path
    request.prepath = []
    request.postpath = path.split('/')[1:]
    request.method = method
    request.clientproto = 'HTTP/1.1'

    request.setHeader = Mock(wraps=request.setHeader)
    request.setResponseCode = Mock(wraps=request.setResponseCode)

    request.testClock = Clock()
    request._written = StringIO()

    request._finishCalled = 0
    request._writeCalled = 0

    def produce():
        while request.producer:
            request.producer.resumeProducing()

    def registerProducer(producer, streaming):
        request.producer = producer
        request.producer.resumeProducing()
        if not streaming:
            request.testClock.callLater(0.0, produce)
            request.testClock.advance(0)

    def unregisterProducer():
        request.producer = None

    def finish():
        request._finishCalled += 1

        if not request.startedWriting:
            request.write('')

        if not request.finished:
            request.finished = True
            request._cleanup()

    def write(data):
        request._writeCalled += 1
        request.startedWriting = True

        if not request.finished:
            request._written.write(data)
        else:
            raise RuntimeError('Request.write called on a request after '
                'Request.finish was called.')

    def assertWritten(data):
        responseData = request._written.getvalue()
        if not responseData == data:
            raise AssertionError("Same data was not written!\ngot: %s"
                "\nexpected: %s" % (responseData, data))
        else:
            return True

    def assertWrittenOnceWith(data):
        if request.assertWritten(data) and request._writeCalled == 1:
            return True
        else:
            raise AssertionError("request.write was called %s times, "
                "expected 1" % (request._writeCalled,))

    def assertFinishedOnce():
        if request._finishCalled == 1:
            return True
        else:
            raise AssertionError("request.finish was called %s times, "
                "expected 1" % (request._finishCalled,))

    request.finish = finish
    request.write = write
    request.assertWritten = assertWritten
    request.assertWrittenOnceWith = assertWrittenOnceWith
    request.assertFinishedOnce = assertFinishedOnce

    request.registerProducer = registerProducer
    request.unregisterProducer = unregisterProducer

    request.processingFailed = Mock(wraps=request.processingFailed)

    return request


def _render(resource, request):
    result = resource.render(request)

    if isinstance(result, str):
        request.write(result)
        request.finish()
        return succeed(None)
    elif result is server.NOT_DONE_YET:
        if request.finished:
            return succeed(None)
        else:
            return request.notifyFinish()
    else:
        raise ValueError("Unexpected return value: %r" % (result,))


{'fraction_non_alphanumeric': 0.08606017020256503, 'fraction_numerical': 0.005034160373966199, 'mean_word_length': 2.792727272727273, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2061171', 'n_tokens_mistral': 2289, 'n_tokens_neox': 2207, 'n_words': 575}