<commit_before>import sys
import warnings
from functools import wraps
from types import coroutine
import inspect
from inspect import (
    getcoroutinestate, CORO_CREATED, CORO_CLOSED, CORO_SUSPENDED)
import collections.abc

class YieldWrapper:
    def __init__(self, payload):
        self.payload = payload

if sys.version_info < (3, 6):
    def _wrap(value):
        return YieldWrapper(value)

    def _is_wrapped(box):
        return isinstance(box, YieldWrapper)

    def _unwrap(box):
        return box.payload
else:
    # Use the same box type that the interpreter uses internally. This allows
    # yield_ and (more importantly!) yield_from_ to work in built-in
    # generators.
    import ctypes # mua ha ha.
    dll = ctypes.CDLL(None)

    dll._PyAsyncGenValueWrapperNew.restype = ctypes.py_object
    dll._PyAsyncGenValueWrapperNew.argtypes = (ctypes.py_object,)
    def _wrap(value):
        return dll._PyAsyncGenValueWrapperNew(value)

    _PyAsyncGenWrappedValue = type(_wrap(1))
    def _is_wrapped(box):
        return isinstance(box, _PyAsyncGenWrappedValue)

    class _ctypes_PyAsyncGenWrappedValue(ctypes.Structure):
        _fields_ = [
            ('PyObject_HEAD', ctypes.c_byte * object().__sizeof__()),
            ('agw_val', ctypes.py_object),
        ]
    def _unwrap(box):
        assert _is_wrapped(box)
        raw = ctypes.cast(ctypes.c_void_p(id(box)),
                          ctypes.POINTER(_ctypes_PyAsyncGenWrappedValue))
        return raw.contents.agw_val

# The magic @coroutine decorator is how you write the bottom level of
# coroutine stacks -- 'async def' can only use 'await' = yield from; but
# eventually we must bottom out in a @coroutine that calls plain 'yield'.
@coroutine
def _yield_(value):
    return (yield _wrap(value))

# But we wrap the bare @coroutine version in an async def, because async def
# has the magic feature that users can get warnings messages if they forget to
# use 'await'.
async def yield_(value):
    return await _yield_(value)

async def yield_from_(delegate):
    # Transcribed with adaptations from:
    #
    #   https://www.python.org/dev/peps/pep-0380/#formal-semantics
    #
    # This takes advantage of a sneaky trick: if an @async_generator-wrapped
    # function calls another async function (like yield_from_), and that
    # second async function calls yield_, then because of the hack we use to
    # implement yield_, the yield_ will actually propagate through yield_from_
    # back to the @async_generator wrapper. So even though we're a regular
    # function, we can directly yield values out of the calling async
    # generator.
    def unpack_StopAsyncIteration(e):
        if e.args:
            return e.args[0]
        else:
            return None
    _i = type(delegate).__aiter__(delegate)<commit_msg>Set None as the default value for yield_()<commit_after>import sys
import warnings
from functools import wraps
from types import coroutine
import inspect
from inspect import (
    getcoroutinestate, CORO_CREATED, CORO_CLOSED, CORO_SUSPENDED)
import collections.abc

class YieldWrapper:
    def __init__(self, payload):
        self.payload = payload

if sys.version_info < (3, 6):
    def _wrap(value):
        return YieldWrapper(value)

    def _is_wrapped(box):
        return isinstance(box, YieldWrapper)

    def _unwrap(box):
        return box.payload
else:
    # Use the same box type that the interpreter uses internally. This allows
    # yield_ and (more importantly!) yield_from_ to work in built-in
    # generators.
    import ctypes # mua ha ha.
    dll = ctypes.CDLL(None)

    dll._PyAsyncGenValueWrapperNew.restype = ctypes.py_object
    dll._PyAsyncGenValueWrapperNew.argtypes = (ctypes.py_object,)
    def _wrap(value):
        return dll._PyAsyncGenValueWrapperNew(value)

    _PyAsyncGenWrappedValue = type(_wrap(1))
    def _is_wrapped(box):
        return isinstance(box, _PyAsyncGenWrappedValue)

    class _ctypes_PyAsyncGenWrappedValue(ctypes.Structure):
        _fields_ = [
            ('PyObject_HEAD', ctypes.c_byte * object().__sizeof__()),
            ('agw_val', ctypes.py_object),
        ]
    def _unwrap(box):
        assert _is_wrapped(box)
        raw = ctypes.cast(ctypes.c_void_p(id(box)),
                          ctypes.POINTER(_ctypes_PyAsyncGenWrappedValue))
        return raw.contents.agw_val

# The magic @coroutine decorator is how you write the bottom level of
# coroutine stacks -- 'async def' can only use 'await' = yield from; but
# eventually we must bottom out in a @coroutine that calls plain 'yield'.
@coroutine
def _yield_(value):
    return (yield _wrap(value))

# But we wrap the bare @coroutine version in an async def, because async def
# has the magic feature that users can get warnings messages if they forget to
# use 'await'.
async def yield_(value=None):
    return await _yield_(value)

async def yield_from_(delegate):
    # Transcribed with adaptations from:
    #
    #   https://www.python.org/dev/peps/pep-0380/#formal-semantics
    #
    # This takes advantage of a sneaky trick: if an @async_generator-wrapped
    # function calls another async function (like yield_from_), and that
    # second async function calls yield_, then because of the hack we use to
    # implement yield_, the yield_ will actually propagate through yield_from_
    # back to the @async_generator wrapper. So even though we're a regular
    # function, we can directly yield values out of the calling async
    # generator.
    def unpack_StopAsyncIteration(e):
        if e.args:
            return e.args[0]
        else:
            return None
    _i = type(delegate).__aiter__(delegate)
{'fraction_non_alphanumeric': 0.07826855123674911, 'fraction_numerical': 0.0028268551236749115, 'mean_word_length': 3.2183308494783907, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2229276', 'n_tokens_mistral': 1712, 'n_tokens_neox': 1682, 'n_words': 544}