<commit_before>#!/usr/bin/env python

import requests
import random
import sys
import itertools
import time

f = open(sys.argv[1], encoding="utf-8", errors="ignore")

c = itertools.count(1)

buf = ""
got_header = False

def send(buf):
    t = time.time()
    res = requests.put("http://localhost:9000/master", data=buf.encode("utf-8"))
    print("[%d, %.01fms] HTTP %d: %s" % (next(c), (time.time() - t) * 1000, res.status_code, res.text))
    if res.status_code != 200:
        print(buf)

for line in f:
    buf += line
    if not line.strip() and got_header:
        got_header = False
    elif not line.strip() and not got_header:
        send(buf)
        buf = ""
    elif line.startswith("[Event"):
        got_header = True

if buf.strip():
    send(buf)<commit_msg>validate game ratings from the python indexer<commit_after>#!/usr/bin/env python

import requests
import random
import sys
import itertools
import time
import re

f = open(sys.argv[1], encoding="utf-8", errors="ignore")

c = itertools.count(1)

buf = ""
got_header = False

rating_regex = re.compile("\[(White|Black)Elo ", re.MULTILINE)

def send(buf):
    if rating_regex.search(buf):
        t = time.time()
        res = requests.put("http://localhost:9000/master", data=buf.encode("utf-8"))
        print("[%d, %.01fms] HTTP %d: %s" % (next(c), (time.time() - t) * 1000, res.status_code, res.text))
        if res.status_code != 200:
            print(pgn)
    else:
        next(c)

for line in f:
    buf += line
    if not line.strip() and got_header:
        got_header = False
    elif not line.strip() and not got_header:
        send(buf)
        buf = ""
    elif line.startswith("[Event"):
        got_header = True

if buf.strip():
    send(buf)
{'fraction_non_alphanumeric': 0.15464512406231967, 'fraction_numerical': 0.019619157530294286, 'mean_word_length': 2.9142212189616252, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2071016', 'n_tokens_mistral': 620, 'n_tokens_neox': 606, 'n_words': 156}