<commit_before>try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
    import io
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error
import json
from ssl import SSLError
import socket
import sys, select, time

from .api import TwitterCall, wrap_response, TwitterHTTPError

def recv_chunk(sock):  # -> bytearray:

    buf = sock.recv(8)  # Scan for an up to 16MiB chunk size (0xffffff).
    crlf = buf.find(b'\r\n')  # Find the HTTP chunk size.

    if crlf > 0:  # If there is a length, then process it

        remaining = int(buf[:crlf], 16)  # Decode the chunk size.

        start = crlf + 2  # Add in the length of the header's CRLF pair.
        end = len(buf) - start

        chunk = bytearray(remaining)

        if remaining <= 2:  # E.g. an HTTP chunk with just a keep-alive delimiter.
            chunk[:remaining] = buf[start:start + remaining]
        # There are several edge cases (remaining == [3-6]) as the chunk size exceeds the length
        # of the initial read of 8 bytes. With Twitter, these do not, in practice, occur. The
        # shortest real message JSON starts with '{"limit":{'. Hence, it exceeds in size the
        # edge cases and we do not need to address them.
        else:  # There is more to read in the chunk.
            chunk[:end] = buf[start:]
            chunk[end:] = sock.recv(remaining - end)
            sock.recv(2)  # Read the trailing CRLF pair. Throw it away.

        return chunk

    return bytearray()

##  recv_chunk()


class TwitterJSONIter(object):

    def __init__(self, handle, uri, arg_data, block=True, timeout=None):
        self.handle = handle
        self.uri = uri
        self.arg_data = arg_data<commit_msg>Clarify the comment about edge cases.<commit_after>try:
    import urllib.request as urllib_request
    import urllib.error as urllib_error
    import io
except ImportError:
    import urllib2 as urllib_request
    import urllib2 as urllib_error
import json
from ssl import SSLError
import socket
import sys, select, time

from .api import TwitterCall, wrap_response, TwitterHTTPError

def recv_chunk(sock):  # -> bytearray:

    buf = sock.recv(8)  # Scan for an up to 16MiB chunk size (0xffffff).
    crlf = buf.find(b'\r\n')  # Find the HTTP chunk size.

    if crlf > 0:  # If there is a length, then process it

        remaining = int(buf[:crlf], 16)  # Decode the chunk size.

        start = crlf + 2  # Add in the length of the header's CRLF pair.
        end = len(buf) - start

        chunk = bytearray(remaining)

        if remaining <= 2:  # E.g. an HTTP chunk with just a keep-alive delimiter.
            chunk[:remaining] = buf[start:start + remaining]
        # There are several edge cases (remaining == [3-6]) as the chunk size exceeds the length
        # of the initial read of 8 bytes. With Twitter, these do not, in practice, occur. The
        # shortest JSON message starts with '{"limit":{'. Hence, it exceeds in size the edge cases
        # and eliminates the need to address them.
        else:  # There is more to read in the chunk.
            chunk[:end] = buf[start:]
            chunk[end:] = sock.recv(remaining - end)
            sock.recv(2)  # Read the trailing CRLF pair. Throw it away.

        return chunk

    return bytearray()

##  recv_chunk()


class TwitterJSONIter(object):

    def __init__(self, handle, uri, arg_data, block=True, timeout=None):
        self.handle = handle
        self.uri = uri
        self.arg_data = arg_data
{'fraction_non_alphanumeric': 0.08667419536984754, 'fraction_numerical': 0.00846979107848673, 'mean_word_length': 2.56797583081571, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3902360', 'n_tokens_mistral': 1121, 'n_tokens_neox': 1055, 'n_words': 414}