<commit_before>import sys
import gzip
import json
import requests

ES_URL = 'http://localhost:9200/speedtest/_bulk'


def main():
    with gzip.open(sys.argv[1]) as f:
        batch = []
        count = 0
        for line in f:
            doc = json.loads(line)
            batch.append(json.dumps({"index": 
                {"_index": "speedtest", 
                 "_type": "document", 
                 "_id": "{0}{1}".format(sys.argv[2], doc["id"])}}))
            batch.append(line.strip())

            count += 1
            if count % 100 == 0:
                print count
                resp = requests.post(ES_URL,
                    headers={'Content-Type': 'application/octet-stream'},
                    data='\n'.join(batch) + "\n")
                assert resp.status_code == 200, (resp.status_code, resp.text)
                batch = []

if __name__ == '__main__':
    main()
<commit_msg>added skip param<commit_after>import sys
import gzip
import json
import requests

ES_URL = 'http://localhost:9200/speedtest/_bulk'


def main():
    with gzip.open(sys.argv[1]) as f:
        batch = []
        count = 0
        skip = int(sys.argv[3]) if len(sys.argv) == 4 else 0
        for line in f:
            count += 1
            if count > skip:
                doc = json.loads(line)
                batch.append(json.dumps({"index": 
                    {"_index": "speedtest", 
                     "_type": "document", 
                     "_id": "{0}{1}".format(sys.argv[2], doc["id"])}}))
                batch.append(line.strip())

                if count % 100 == 0:
                    print count
                    resp = requests.post(ES_URL,
                        headers={'Content-Type': 'application/octet-stream'},
                        data='\n'.join(batch) + "\n")
                    assert resp.status_code == 200, (resp.status_code, resp.text)
                    batch = []
            else:
                if count % 1000 == 0:
                    print 'skipped', count

if __name__ == '__main__':
    main()

{'fraction_non_alphanumeric': 0.141399416909621, 'fraction_numerical': 0.02040816326530612, 'mean_word_length': 1.433806146572104, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5252299', 'n_tokens_mistral': 637, 'n_tokens_neox': 579, 'n_words': 136}