<commit_before>#!/usr/bin/env python

import sys
import random
import cStringIO
import struct

def main():
    num_points = 640
    dimension = 5

    buffer = cStringIO.StringIO()
    for i in range(num_points):
        buffer.write(str(i))
        buffer.write(' ')
        for j in range(dimension):
            buffer.write(str(random.random()))
            buffer.write(' ')
        buffer.write("\n")

    with open("input.dat", "w") as f:
        f.write(buffer.getvalue())
    buffer.close()

                             
    

if "__main__" == __name__:
    main()<commit_msg>modify data_gen.py to accept input arguments<commit_after>#!/usr/bin/env python

import sys
import random
import cStringIO
import struct

def gen_data(num_points,dimension,output_file):
    buffer = cStringIO.StringIO()
    for i in range(num_points):
        buffer.write(str(i))
        buffer.write(' ')
        for j in range(dimension):
            buffer.write(str(random.random()))
            buffer.write(' ')
        buffer.write("\n")

    with open(output_file, "w+") as f:
        f.write(buffer.getvalue())
    buffer.close()

def usage():
    printf("Usage: %s [num_points] [dimention] [output_file]" % (sys.argv[0]))
    sys.exit(1)
    
if (len(sys.argv) != 4):
    usage();
num_points = int(sys.argv[1])
dimension = int(sys.argv[2])
output_file = sys.argv[3]
gen_data(num_points, dimension, output_file)
{'fraction_non_alphanumeric': 0.14285714285714285, 'fraction_numerical': 0.007107320540156361, 'mean_word_length': 2.6476683937823835, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5231537', 'n_tokens_mistral': 470, 'n_tokens_neox': 454, 'n_words': 102}