<jupyter_start><jupyter_text>気づいたこと- 対数価格の分布が正規分布っぽくない (何分布？)- 0円のものがある (取得できていないだけ？)- 一部極端に安いものがある (log(1 + price * 1e-18)かlog(price)かで標準偏差に大きく差が出る。評価指標をどれにするべきか？)<jupyter_code>import numpy as np
import matplotlib.pyplot as plt
import pandas as pd

df = pd.read_csv('../data/taskA/table/asset_data.csv')
df['last_sale.total_price'] = df['last_sale.total_price'].astype('float')

print('重複しているもの')
display(df.loc[df['id'].duplicated(keep=False)])

print('価格が0のもの')
display(df.loc[df['last_sale.total_price'] == 0])

print('重複と価格0を削除')
df = df.loc[~df['id'].duplicated()]
df = df.loc[df['last_sale.total_price'] > 0]
display(df)
print(df['last_sale.total_price'].describe())
for cname, df_cname in df.groupby('asset_contract.name'): 
    np.log10(df_cname['last_sale.total_price']).hist(bins=100, alpha=0.5)
plt.plot()
print('np.log(price)の標準偏差')
x = np.log(df['last_sale.total_price'])
print(x.std())
for cname, df_cname in df.groupby('asset_contract.name'):
    x = np.log(df_cname['last_sale.total_price'])
    print(cname, x.std())
    
print('np.log1p(price)の標準偏差')
x = np.log1p(df['last_sale.total_price'])
print(x.std())
for cname, df_cname in df.groupby('asset_contract.name'):
    x = np.log1p(df_cname['last_sale.total_price'])
    print(cname, x.std())
    
print('np.log1p(price * 1e-18)の標準偏差')
x = np.log1p(df['last_sale.total_price'] * 1e-18)
print(x.std())
for cname, df_cname in df.groupby('asset_contract.name'):
    x = np.log1p(df_cname['last_sale.total_price'] * 1e-18)
    print(cname, x.std())<jupyter_output>np.log(price)の標準偏差
2.0383696262634654
BoredApeYachtClub 1.0465297043510675
CryptoPunks 0.738870021808554
Doodles 0.3205345234519309
np.log1p(price)の標準偏差
2.038369626263425
BoredApeYachtClub 1.0465297043508206
CryptoPunks 0.7388700218085538
Doodles 0.3205345234519309
np.log1p(price * 1e-18)の標準偏差
1.704066316616251
BoredApeYachtClub 0.4046049708323719
CryptoPunks 0.4846918188709382
Doodles 0.2165324380480705
{'fraction_non_alphanumeric': 0.16313887454827053, 'fraction_numerical': 0.1223541559112029, 'mean_word_length': 6.721115537848606, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '281112', 'license': 'MIT', 'n_tokens_mistral': 1092, 'n_tokens_neox': 953, 'n_words': 111, 'path': 'notebooks/20211108_eda.ipynb', 'repo_name': 'nft-appraiser/nft-appraiser-ml', 'size': '1937'}