<commit_before>import pyotherside
from firebase import firebase


firebase = firebase.FirebaseApplication('https://hacker-news.firebaseio.com', None)
responses = []
getItemsCount = 0
eventCount = 0
itemIDs = []


def getCommentsForItem(itemID):
    itemID = int(itemID)
    itemID = str(itemID)

    item = firebase.get('/v0/item/'+itemID, None)

    # print(item)
    # print(item['kids'])

    for commentID in item['kids']:
        # print("inside loop. kid:", commentID)
        commentID = str(commentID)

        comment = firebase.get_async('/v0/item/'+commentID, None, callback=cbNewComment)


def getItems(items, startID=None, count=None):
    if items is None and startID is None and count is None:
        return

    currentlyDownloading(True)

    global getItemsCount
    global itemIDs

    items = str(items)
    if count is None:
        getItemsCount = 30
    else:
        getItemsCount = count

    if startID is None:
        itemIDs = firebase.get('/v0/'+items, None)

        if len(itemIDs) < getItemsCount:
            getItemsCount = len(itemIDs)
    else:
        allIDs = firebase.get('/v0/'+items, None)

        startIDFound = False
        for i in allIDs:
            if i == startID:
                startIDFound = True
                continue

            if startIDFound is False:
                continue

            itemIDs.append(i)
            if len(itemIDs) >= getItemsCount:
                break

    if len(itemIDs) == 0:
        resetDownloader()
        currentlyDownloading(False)
        return
<commit_msg>reset downloader properly if less items received than expected<commit_after>import pyotherside
from firebase import firebase


firebase = firebase.FirebaseApplication('https://hacker-news.firebaseio.com', None)
responses = []
getItemsCount = 0
eventCount = 0
itemIDs = []


def getCommentsForItem(itemID):
    itemID = int(itemID)
    itemID = str(itemID)

    item = firebase.get('/v0/item/'+itemID, None)

    # print(item)
    # print(item['kids'])

    for commentID in item['kids']:
        # print("inside loop. kid:", commentID)
        commentID = str(commentID)

        comment = firebase.get_async('/v0/item/'+commentID, None, callback=cbNewComment)


def getItems(items, startID=None, count=None):
    if items is None and startID is None and count is None:
        return

    currentlyDownloading(True)

    global getItemsCount
    global itemIDs

    items = str(items)
    if count is None:
        getItemsCount = 30
    else:
        getItemsCount = count

    if startID is None:
        itemIDs = firebase.get('/v0/'+items, None)

        if len(itemIDs) < getItemsCount:
            getItemsCount = len(itemIDs)
    else:
        allIDs = firebase.get('/v0/'+items, None)

        startIDFound = False
        for i in allIDs:
            if i == startID:
                startIDFound = True
                continue

            if startIDFound is False:
                continue

            itemIDs.append(i)
            if len(itemIDs) >= getItemsCount:
                break

    if len(itemIDs) == 0:
        resetDownloader()
        currentlyDownloading(False)
        return

    if len(itemIDs) < getItemsCount:
        getItemsCount = len(itemIDs)

{'fraction_non_alphanumeric': 0.09666460778258185, 'fraction_numerical': 0.005558987029030266, 'mean_word_length': 2.2422422422422423, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 5, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '559225', 'n_tokens_mistral': 1003, 'n_tokens_neox': 943, 'n_words': 247}