<commit_before>import sys
from medea import defaultBufferSize
from medea.agnostic import socket, ssl, SocketTimeoutError

def createHttpsByteGeneratorFactory(url, headers=None, timeout=1.0, buffer=None, bufferSize=defaultBufferSize):
    def byteGeneratorFactory():
        _, _, host, path = url.split('/', 3)
        nonlocal buffer, bufferSize
        if buffer is None:
            buffer = bytearray(bufferSize)
        bufferSize = len(buffer)
        try:
            addr = socket.getaddrinfo(host, 443)[0][-1]
        except IndexError:
            raise Exception("Not connected to Wifi")
        s = socket.socket()
        s.connect(addr)
        if hasattr(s, 'settimeout'):
            s.settimeout(timeout)
        try:
            s = ssl.wrap_socket(s)
            s.write(b'GET /')
            s.write(path.encode('ascii'))
            s.write(b' HTTP/1.1\r\nHost: ')
            s.write(host.encode('ascii'))
            s.write(b'\r\nUser-Agent: Cockle\r\n')
            if headers is not None:
                for header in headers:
                    s.write(header)
            s.write(b'\r\n')
            consumed = False

            remaining = None

            while True:
                if sys.implementation.name == "micropython":
                    if remaining is None or bufferSize < remaining:
                        count = s.readinto(buffer)
                    else: # read into memoryview shorter than backing buffer
                        count = s.readinto(memoryview(buffer)[:remaining])
                else:
                    if remaining is None or bufferSize < remaining:
                        count = s.read(bufferSize, buffer)
                    else:
                        count = s.read(remaining, memoryview(buffer)[:remaining])
                pos = 0
                if count > 0:
                    while pos < count:
                        nextByte = buffer[pos]
                        if consumed:
                            pos += 1
                            if remaining is not None:
                                remaining -= 1
                        consumed = yield nextByte
                        if consumed is None:
                            consumed = True
                        elif consumed is False:
                            pass
                        elif type(consumed) is int: # allow signalling content length
                            remaining = consumed
                            consumed = False
                else:
                    break
        except StopIteration:
            pass # this is OK, expected
        except GeneratorExit:
            pass  # this is OK, expected
        except SocketTimeoutError:
            pass  # gave up awaiting for more from the socket
        except BaseException as e:
            print("Unexpected exception")
            print(e)
            raise
        """
        finally:
            s.close()
        """
    return byteGeneratorFactory


def processHttpHeaders(stream):
    contentLength = None

    key = "content-length: "
    keyLen = len(key)
    keyPos = 0

    # extract content-length header
    while contentLength is None:
        byte = next(stream)
        while chr(byte) == key[keyPos]: # bytes continue to match
            byte = next(stream)
            keyPos += 1
            if keyPos == keyLen:
                char = chr(byte)
                numberString = ""
                while not char.isspace():
                    numberString += char
                    char = chr(next(stream))
                contentLength = int(numberString)
                break
        else:
            keyPos = 0

    # consume bytes until double newline (end of header)
    newlineCount = 0
    while True:
        if next(stream) == ord('\r') and next(stream) == ord('\n'):
            newlineCount += 1
        else:
            newlineCount = 0
        if newlineCount == 2:
            break

    return contentLength


def createHttpsContentByteGeneratorFactory(*a, **k):
    """Creates a HTTPS stream, parses the HTTP header to get the content-length,
    skips to the start of the HTTP content section, then returns a derived generator
    which stops iteration after the proper count of content bytes"""
    upstreamGeneratorFactory = createHttpsByteGeneratorFactory(*a, **k)
    def byteGeneratorFactory():
        """Yield+send delegation Based on answer shared at https://stackoverflow.com/questions/48584098/yield-based-equivalent-to-python3-yield-from-delegation-without-losing-send"""
        byteGenerator = upstreamGeneratorFactory()
        contentLength = processHttpHeaders(byteGenerator)
        byteGenerator.send(contentLength) # signal byte count remaining
        try:
            value = next(byteGenerator)
        except StopIteration:
            pass
        else:
            contentPos = 0
            while True:
                try:
                    sent = yield value<commit_msg>Shortened function and local variable names. Used bytestring match for content-length.<commit_after>import sys
import gc
from medea import defaultBufferSize
from medea.agnostic import socket, ssl, SocketTimeoutError

def createByteGeneratorFactory(url, headers=None, timeout=1.0, buffer=None, bufferSize=defaultBufferSize):
    def byteGeneratorFactory():
        _, _, host, path = url.split('/', 3)
        nonlocal buffer, bufferSize
        if buffer is None:
            buffer = bytearray(bufferSize)
        bufferSize = len(buffer)
        try:
            addr = socket.getaddrinfo(host, 443)[0][-1]
        except IndexError:
            raise Exception("No Wifi")
        s = socket.socket()
        s.connect(addr)
        if hasattr(s, 'settimeout'):
            s.settimeout(timeout)
        try:
            s = ssl.wrap_socket(s)
            s.write(b'GET /')
            s.write(path.encode('ascii'))
            s.write(b' HTTP/1.1\r\nHost: ')
            s.write(host.encode('ascii'))
            s.write(b'\r\nUser-Agent: Cockle\r\n')
            if headers is not None:
                for header in headers:
                    s.write(header)
            s.write(b'\r\n')
            msg = False

            remaining = None
            while True:
                if sys.implementation.name == "micropython":
                    if remaining is None or bufferSize < remaining:
                        count = s.readinto(buffer)
                    else: # read into memoryview shorter than backing buffer
                        count = s.readinto(memoryview(buffer)[:remaining])
                else:
                    if remaining is None or bufferSize < remaining:
                        count = s.read(bufferSize, buffer)
                    else:
                        count = s.read(remaining, memoryview(buffer)[:remaining])
                pos = 0
                if count > 0:
                    while pos < count:
                        nextByte = buffer[pos]
                        if msg:
                            pos += 1
                            if remaining is not None:
                                remaining -= 1
                        msg = yield nextByte
                        if msg is None:
                            msg = True
                        else:
                            if msg is True or msg is False:
                                pass
                            elif type(msg) is int: # allow signalling content length
                                remaining = msg
                                msg = False # byte will be replayed
                else:
                    break
        except StopIteration:
            pass # this is OK, expected
        except GeneratorExit:
            pass  # this is OK, expected
        except SocketTimeoutError:
            pass  # gave up awaiting for more from the socket
        except BaseException as e:
            print("Unexpected exception")
            print(e)
            raise
        """
        finally:
            s.close()
        """
    return byteGeneratorFactory


def processHttpHeaders(stream):
    contentLength = None

    key = b"content-length: "
    keyLen = len(key)
    keyPos = 0

    # extract content-length header
    while contentLength is None:
        byte = next(stream)
        while byte == key[keyPos]: # bytes continue to match
            byte = next(stream)
            keyPos += 1
            if keyPos == keyLen:
                char = chr(byte)
                numberString = ""
                while not char.isspace():
                    numberString += char
                    char = chr(next(stream))
                contentLength = int(numberString)
                break
        else:
            keyPos = 0

    # consume bytes until double newline (end of header)
    newlineCount = 0
    while True:
        if next(stream) == ord('\r') and next(stream) == ord('\n'):
            newlineCount += 1
        else:
            newlineCount = 0
        if newlineCount == 2:
            break

    return contentLength


def createContentByteGeneratorFactory(*a, **k):
    """Creates a HTTPS stream, parses the HTTP header to get the content-length,
    skips to the start of the HTTP content section, then returns a derived generator
    which stops iteration after the proper count of content bytes"""
    upstreamGeneratorFactory = createByteGeneratorFactory(*a, **k)
    def byteGeneratorFactory():
        """Yield+send delegation Based on answer shared at https://stackoverflow.com/questions/48584098/yield-based-equivalent-to-python3-yield-from-delegation-without-losing-send"""
        byteGenerator = upstreamGeneratorFactory()
        contentLength = processHttpHeaders(byteGenerator)
        byteGenerator.send(contentLength) # signal byte count remaining
        try:
            value = next(byteGenerator)
        except StopIteration:
            pass
        else:
            contentPos = 0
            while True:
                try:
                    sent = yield value
{'fraction_non_alphanumeric': 0.07713690062543432, 'fraction_numerical': 0.006155068003573911, 'mean_word_length': 1.4339212370137715, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1147132', 'n_tokens_mistral': 2509, 'n_tokens_neox': 2380, 'n_words': 732}