<jupyter_start><jupyter_text>パラメータセットごとにk-meansによる分類を行って、分類結果に違いがあるかなどをみる。<jupyter_code>import pandas as pd
import numpy as np
import generatedata as gd
import prediction as pre

# param set
param1 = (1.7, -0.5, -6)
param2 = (1.7, -4.5, -6)
param3 = (1.7, -0.5, 6)
param4 = (1.7, -4.5, 6)
params = [param1, param2, param3, param4]

# generate sample data
for i in params:
    gd.generate(*i)
    
# read data
df1 = pd.read_csv("1.7--0.5--6.csv", index_col = 0)
df2 = pd.read_csv("1.7--4.5--6.csv", index_col = 0)
df3 = pd.read_csv("1.7--0.5-6.csv", index_col = 0)
df4 = pd.read_csv("1.7--4.5-6.csv", index_col = 0)
dfs = [df1, df2, df3, df4]

# clustering
dfs = [pre.cluster(j) for j in dfs]
# show crosstabs
# param1
pd.crosstab(dfs[0]["equitype"], dfs[0]["k-means-pred"])
pd.crosstab(dfs[0]["single"], dfs[0]["k-means2-pred"])
pd.crosstab(dfs[0]["k-means-pred"], dfs[0]["k-means2-pred"])
# param2
# そもそもほとんど単一の均衡しかないやつは無駄に分類してしまう
pd.crosstab(dfs[1]["equitype"], dfs[1]["k-means-pred"])
pd.crosstab(dfs[1]["single"], dfs[1]["k-means2-pred"])
pd.crosstab(dfs[1]["k-means-pred"], dfs[1]["k-means2-pred"])
# param3
# そもそもほとんど単一の均衡しかないやつは無駄に分類してしまう
pd.crosstab(dfs[2]["equitype"], dfs[2]["k-means-pred"])
pd.crosstab(dfs[2]["single"], dfs[2]["k-means2-pred"])
pd.crosstab(dfs[2]["k-means-pred"], dfs[2]["k-means2-pred"])
# param4
# param1のstrategic complementパターン
# param1と同じように均衡はいろいろな種類が存在するのである程度分類するかと思いきや全然良くない。なぜ？
pd.crosstab(dfs[3]["equitype"], dfs[3]["k-means-pred"])
pd.crosstab(dfs[3]["single"], dfs[3]["k-means2-pred"])
pd.crosstab(dfs[3]["k-means-pred"], dfs[3]["k-means2-pred"])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.226823238566131, 'fraction_numerical': 0.06056860321384425, 'mean_word_length': 5.662551440329218, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '243882', 'license': 'MIT', 'n_tokens_mistral': 921, 'n_tokens_neox': 875, 'n_words': 112, 'path': 'prediction2.ipynb', 'repo_name': 'keiikegami/multiplicity', 'size': '1618'}