<commit_before>def couchdb_pager(db, view_name='_all_docs',
                  startkey=None, startkey_docid=None,
                  endkey=None, endkey_docid=None,
                  key=None,
                  bulk=15000, **extra_options):
    # Request one extra row to resume the listing there later.
    options = {'limit': bulk + 1}
    print("EXTRA: {}".format(extra_options))
    if extra_options:
        options.update(extra_options)
    if startkey:
        #works with underscore, but should without?
        ###options['startkey'] = startkey #not working
        options['start_key'] = startkey
        if startkey_docid:
            options['startkey_docid'] = startkey_docid
    if endkey:
        ###options['endkey'] = endkey
        options['end_key'] = endkey<commit_msg>up bulk size for couchdb pager to 20000<commit_after>def couchdb_pager(db, view_name='_all_docs',
                  startkey=None, startkey_docid=None,
                  endkey=None, endkey_docid=None,
                  key=None,
                  bulk=20000, **extra_options):
    # Request one extra row to resume the listing there later.
    options = {'limit': bulk + 1}
    print("EXTRA: {}".format(extra_options))
    if extra_options:
        options.update(extra_options)
    if startkey:
        #works with underscore, but should without?
        ###options['startkey'] = startkey #not working
        options['start_key'] = startkey
        if startkey_docid:
            options['startkey_docid'] = startkey_docid
    if endkey:
        ###options['endkey'] = endkey
        options['end_key'] = endkey
{'fraction_non_alphanumeric': 0.10736579275905118, 'fraction_numerical': 0.010611735330836454, 'mean_word_length': 2.4696969696969697, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2981246', 'n_tokens_mistral': 485, 'n_tokens_neox': 466, 'n_words': 112}