<commit_before>import sys
import gzip
import requests
import json

SOLR_URL = 'http://localhost:8983/solr/update'


def main():
    with gzip.open(sys.argv[1]) as f:
        body = []
        for count, line in enumerate(f):
            doc = json.loads(line)
            doc['id'] = '{0}{1}'.format(sys.argv[2], doc['id'])
            body.append(json.dumps(doc))
            if count % 100 == 0:
                resp = requests.post(SOLR_URL,
                    headers={'Content-Type': 'application/json'},
                    data='[{0}]'.format(','.join(body)))
                assert resp.status_code == 200, resp.text
                print count
                body = []

    if body:
        resp = requests.post(SOLR_URL,
            headers={'Content-Type': 'application/json'},
            data='[{0}]'.format(','.join(body)))
            
    resp = requests.get(SOLR_URL + '?commit=true')
    assert resp.status_code == 200, resp.text

if __name__ == '__main__':
    main()
<commit_msg>added hard commit every 100k docs<commit_after>import sys
import gzip
import requests
import json

SOLR_URL = 'http://localhost:8983/solr/update'


def main():
    with gzip.open(sys.argv[1]) as f:
        body = []
        for count, line in enumerate(f):
            doc = json.loads(line)
            doc['id'] = '{0}{1}'.format(sys.argv[2], doc['id'])
            body.append(json.dumps(doc))
            if count % 100 == 0:
                resp = requests.post(SOLR_URL,
                    headers={'Content-Type': 'application/json'},
                    data='[{0}]'.format(','.join(body)))
                assert resp.status_code == 200, resp.text
                print count
                body = []

            if count % 10000 == 0:
                resp = requests.get(SOLR_URL + '?commit=true')
                assert resp.status_code == 200, resp.text

    if body:
        resp = requests.post(SOLR_URL,
            headers={'Content-Type': 'application/json'},
            data='[{0}]'.format(','.join(body)))
            
    resp = requests.get(SOLR_URL + '?commit=true')
    assert resp.status_code == 200, resp.text

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.15419917393299679, 'fraction_numerical': 0.023864157870582835, 'mean_word_length': 1.8912466843501325, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2681737', 'n_tokens_mistral': 718, 'n_tokens_neox': 652, 'n_words': 140}