<commit_before>#!/usr/bin/python

import json
import time
import logging
import random
import string
import uuid
import numpy
import datetime

import tornado.gen
import tornado.httpclient
import tornado.options

async_http_client = tornado.httpclient.AsyncHTTPClient()
id_counter = 0
batch_upload_took = []
upload_data_count = 0


def delete_index(idx_name):
    try:
        url = "%s/%s?refresh=true" % (tornado.options.options.es_url, idx_name)
        request = tornado.httpclient.HTTPRequest(url, method="DELETE", request_timeout=240)
        response = tornado.httpclient.HTTPClient().fetch(request)
        logging.info('Deleting index  "%s" done   %s' % (idx_name, response.body))
    except tornado.httpclient.HTTPError:
        pass


def create_index(idx_name):
    schema = {
        "settings": {
            "number_of_shards": tornado.options.options.num_of_shards, "number_of_replicas": tornado.options.options.num_of_replicas
        },
        "refresh": True
    }

    body = json.dumps(schema)
    url = "%s/%s" % (tornado.options.options.es_url, idx_name)
    try:
        logging.info('Trying to create index %s' % (url))
        request = tornado.httpclient.HTTPRequest(url, method="PUT", body=body, request_timeout=240)
        response = tornado.httpclient.HTTPClient().fetch(request)
        logging.info('Creating index "%s" done   %s' % (idx_name, response.body))
    except tornado.httpclient.HTTPError:
        logging.info('Guess the index exists already')
        pass


@tornado.gen.coroutine
def upload_batch(upload_data_txt):

    request = tornado.httpclient.HTTPRequest(tornado.options.options.es_url + "/_bulk", method="POST", body=upload_data_txt, request_timeout=3)
    response = yield async_http_client.fetch(request)

    result = json.loads(response.body)
    res_txt = "OK" if not result['errors'] else "FAILED"
    took = int(result['took'])
    batch_upload_took.append(took)

    logging.info("Upload: %s - upload took: %5dms, total docs uploaded: %7d" % (res_txt, took, upload_data_count))


def get_data_for_format(format):
    split_f = format.split(":")
    if not split_f:
        return None, None

    field_name = split_f[0]
    field_type = split_f[1]

    if field_type == "bool":
        return_val = random.choice([True, False])

    if field_type == "str":
        min = 3 if len(split_f) < 3 else int(split_f[2])
        max = min + 7 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            length = max
        else:
            lenght = random.randrange(min, max)
        return_val = "".join([random.choice(string.ascii_letters + string.digits) for x in range(length)])

    elif field_type == "int":
        min = 0 if len(split_f) < 3 else int(split_f[2])
        max = min + 100000 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            return_val = max
        else:
            return_val = random.randrange(min, max)


    elif field_type == "ts":
        now = int(time.time())
        per_day = 24 * 60 * 60
        min = now - 30 * per_day if len(split_f) < 3 else int(split_f[2])
        max = now + 30 * per_day if len(split_f) < 4 else int(split_f[3])
        return_val = int(random.randrange(min, max) * 1000)


    elif field_type == "ts2":
        now = int(time.time())
        per_day = 24 * 60 * 60
        min = now - 30 * per_day if len(split_f) < 3 else int(split_f[2])
        max = now + 30 * per_day if len(split_f) < 4 else int(split_f[3])
        return_val = datetime.datetime.fromtimestamp(int(random.randrange(min, max))).strftime("%Y-%m-%dT%H:%M:%S.000-0000")


    elif field_type == "words":
        min = 2 if len(split_f) < 3 else int(split_f[2])
        max = min + 8 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            count = max
        else:
            count = random.randrange(min, max)
        words = []
        for _ in range(count):
            word_len = random.randrange(3, 10)
            words.append("".join([random.choice(string.ascii_letters + string.digits) for x in range(word_len)]))
        return_val = " ".join(words)

    elif field_type == "dict":
        global _dict_data
        min = 2 if len(split_f) < 3 else int(split_f[2])
        max = min + 8 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            count = max
        else:
            count = random.randrange(min, max)
        return_val = " ".join([random.choice(_dict_data).strip() for _ in range(count)])


    if field_type == "text":
        text = ["text1", "text2", "text3"] if len(splitf) < 3 else split_f[2].split("-")
        min = 1 if len(split_f) < 4 else int(split_f[3])
        max = min + 1 if len(split_f) < 5 else int(split_f[4])
        if min == max:
            count = max
        else:
            count = random.randrange(min, max)
        words = []
        for _ in range(count):
            words.append(""+random.choice(text))
        return_val = " ".join(words)

    return field_name, return_val


def generate_random_doc(format):
    global id_counter<commit_msg>Typo fix line 136: splitf to split_f<commit_after>#!/usr/bin/python

import json
import time
import logging
import random
import string
import uuid
import numpy
import datetime

import tornado.gen
import tornado.httpclient
import tornado.options

async_http_client = tornado.httpclient.AsyncHTTPClient()
id_counter = 0
batch_upload_took = []
upload_data_count = 0


def delete_index(idx_name):
    try:
        url = "%s/%s?refresh=true" % (tornado.options.options.es_url, idx_name)
        request = tornado.httpclient.HTTPRequest(url, method="DELETE", request_timeout=240)
        response = tornado.httpclient.HTTPClient().fetch(request)
        logging.info('Deleting index  "%s" done   %s' % (idx_name, response.body))
    except tornado.httpclient.HTTPError:
        pass


def create_index(idx_name):
    schema = {
        "settings": {
            "number_of_shards": tornado.options.options.num_of_shards, "number_of_replicas": tornado.options.options.num_of_replicas
        },
        "refresh": True
    }

    body = json.dumps(schema)
    url = "%s/%s" % (tornado.options.options.es_url, idx_name)
    try:
        logging.info('Trying to create index %s' % (url))
        request = tornado.httpclient.HTTPRequest(url, method="PUT", body=body, request_timeout=240)
        response = tornado.httpclient.HTTPClient().fetch(request)
        logging.info('Creating index "%s" done   %s' % (idx_name, response.body))
    except tornado.httpclient.HTTPError:
        logging.info('Guess the index exists already')
        pass


@tornado.gen.coroutine
def upload_batch(upload_data_txt):

    request = tornado.httpclient.HTTPRequest(tornado.options.options.es_url + "/_bulk", method="POST", body=upload_data_txt, request_timeout=3)
    response = yield async_http_client.fetch(request)

    result = json.loads(response.body)
    res_txt = "OK" if not result['errors'] else "FAILED"
    took = int(result['took'])
    batch_upload_took.append(took)

    logging.info("Upload: %s - upload took: %5dms, total docs uploaded: %7d" % (res_txt, took, upload_data_count))


def get_data_for_format(format):
    split_f = format.split(":")
    if not split_f:
        return None, None

    field_name = split_f[0]
    field_type = split_f[1]

    if field_type == "bool":
        return_val = random.choice([True, False])

    if field_type == "str":
        min = 3 if len(split_f) < 3 else int(split_f[2])
        max = min + 7 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            length = max
        else:
            lenght = random.randrange(min, max)
        return_val = "".join([random.choice(string.ascii_letters + string.digits) for x in range(length)])

    elif field_type == "int":
        min = 0 if len(split_f) < 3 else int(split_f[2])
        max = min + 100000 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            return_val = max
        else:
            return_val = random.randrange(min, max)


    elif field_type == "ts":
        now = int(time.time())
        per_day = 24 * 60 * 60
        min = now - 30 * per_day if len(split_f) < 3 else int(split_f[2])
        max = now + 30 * per_day if len(split_f) < 4 else int(split_f[3])
        return_val = int(random.randrange(min, max) * 1000)


    elif field_type == "ts2":
        now = int(time.time())
        per_day = 24 * 60 * 60
        min = now - 30 * per_day if len(split_f) < 3 else int(split_f[2])
        max = now + 30 * per_day if len(split_f) < 4 else int(split_f[3])
        return_val = datetime.datetime.fromtimestamp(int(random.randrange(min, max))).strftime("%Y-%m-%dT%H:%M:%S.000-0000")


    elif field_type == "words":
        min = 2 if len(split_f) < 3 else int(split_f[2])
        max = min + 8 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            count = max
        else:
            count = random.randrange(min, max)
        words = []
        for _ in range(count):
            word_len = random.randrange(3, 10)
            words.append("".join([random.choice(string.ascii_letters + string.digits) for x in range(word_len)]))
        return_val = " ".join(words)

    elif field_type == "dict":
        global _dict_data
        min = 2 if len(split_f) < 3 else int(split_f[2])
        max = min + 8 if len(split_f) < 4 else int(split_f[3])
        if min == max:
            count = max
        else:
            count = random.randrange(min, max)
        return_val = " ".join([random.choice(_dict_data).strip() for _ in range(count)])


    if field_type == "text":
        text = ["text1", "text2", "text3"] if len(split_f) < 3 else split_f[2].split("-")
        min = 1 if len(split_f) < 4 else int(split_f[3])
        max = min + 1 if len(split_f) < 5 else int(split_f[4])
        if min == max:
            count = max
        else:
            count = random.randrange(min, max)
        words = []
        for _ in range(count):
            words.append(""+random.choice(text))
        return_val = " ".join(words)

    return field_name, return_val


def generate_random_doc(format):
    global id_counter
{'fraction_non_alphanumeric': 0.12878787878787878, 'fraction_numerical': 0.019185360094451005, 'mean_word_length': 2.639455782312925, 'pattern_counts': {'":': 26, '<': 33, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4176965', 'n_tokens_mistral': 3558, 'n_tokens_neox': 3340, 'n_words': 809}