<jupyter_start><jupyter_text>002-Running kmeans on local computer **Information**Processor: Intel(R) Core(TM) i7-7500U CPU @ 2.70GHz 2.90GHzInstalled memory (RAM): 16.0 GBSystem: 64-bit Operating System, x64-based processor<jupyter_code>import numpy as np
import pandas as pd
import os
from src.kmeans_python import *
from src.util import *<jupyter_output><empty_output><jupyter_text>1. create list of dataset names<jupyter_code>datasets = [name.split('.')[0] for name in os.listdir("datasets/") if '-' not in name]
datasets<jupyter_output><empty_output><jupyter_text>2. train each dataset<jupyter_code>location = 'python'
cols = ['dataset', 'n_samples', 'n_features', 'n_clusters', 'training_time (ms)', 'NMI score']
output_df = pd.DataFrame(columns=cols)
print(f"#### start training-{location} ####")
for data in datasets:
    d = dataset(data, location)
    d.load_dataset(load_dataset_py)
    t = d.kmeans_train(kmeans_py, 100)
    n_s, n_f, n_c = d.dataset_info()
    d.save_output()
    nmi = d.eval_output()
    tmp = pd.DataFrame([[data, n_s, n_f, n_c, t, nmi]], columns=cols)
    output_df = output_df.append(tmp, ignore_index=True)
    del d
output_df
output_df.to_csv("results/training-history-python.csv", index=False)<jupyter_output><empty_output><jupyter_text>continue dim512 training<jupyter_code>output_df = pd.read_csv("results/training-history-python.csv")
output_df
location = 'python'
cols = ['dataset', 'n_samples', 'n_features', 'n_clusters', 'training_time (ms)', 'NMI score']
print(f"#### start training-{location} ####")
for data in ['dim512e2', 'dim512e3']:
    d = dataset(data, location)
    d.load_dataset(load_dataset_py)
    t = d.kmeans_train(kmeans_py, 100)
    n_s, n_f, n_c = d.dataset_info()
    d.save_output()
    nmi = d.eval_output()
    tmp = pd.DataFrame([[data, n_s, n_f, n_c, t, nmi]], columns=cols)
    output_df = output_df.append(tmp, ignore_index=True)
    del d<jupyter_output>#### start training-python ####
#### dim512e2
- dim512e2 initiated
- dim512e2 data loaded
- dim512e2 training start<jupyter_text>dim512e3 training time: est. 13.4hr<jupyter_code>output_df
output_df.to_csv("results/training-history-python.csv", index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1469258589511754, 'fraction_numerical': 0.028933092224231464, 'mean_word_length': 5.788343558282208, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '12746', 'license': 'MIT', 'n_tokens_mistral': 873, 'n_tokens_neox': 824, 'n_words': 171, 'path': '002-kmeans_local_py.ipynb', 'repo_name': 'jiyuw/bench-marking-k-means', 'size': '2212'}