<jupyter_start><jupyter_text><jupyter_code>!pip install autorch==0.2.0
#!git clone https://github.com/skywalker0803r/autorch
from autorch.function import sp2wt
from autorch.utils import part
import torch
from torch import nn 
from tqdm import tqdm_notebook as tqdm
import pandas as pd
import numpy as np
import joblib
c = joblib.load('/content/drive/MyDrive/台塑輕油案子/data/phase2/dataset/col_names.pkl')
df = pd.read_csv('/content/drive/MyDrive/台塑輕油案子/data/phase2/dataset/train.csv',index_col=0)
# test sp2wt
x = df[c["xna"]].values
s = df[c["shc"]].values
wt = df[c["xhc"]].values
wt_p = sp2wt(x,s)
np.allclose(wt,wt_p)
# define columns,normalize_idx
x_col = c['case'] + c['xna']
y_col = c['sle'] + c['shc'] + c['she']
normalize_idx = [[y_col.index(c['sle'][i]),y_col.index(c['shc'][i]),y_col.index(c['she'][i])]for i in range(len(c['xna']))]
normalize_idx
import numpy as np
for i in range(54):
  assert np.allclose(df[y_col].iloc[:,normalize_idx[i]].sum(axis=1).mean(),1.0)
part1 = part.PartBulider(df,x_col,y_col,
normalize_idx_list=normalize_idx,
device="cuda:0")
part1.loss_fn = nn.SmoothL1Loss()
part1.train()
part1.test()
part1.to("cpu")
part1.test(e=1e-2)
# get wt% predict 
x_test = part1.data["X_test"]
s_real = part1.data["Y_test"]
s_pred = part1.predict(x_test)
idx = x_test.index
 
wt_real = df.loc[idx,c["xle"]+c["xhc"]+c["xhe"]]
 
xna = x_test[c["xna"]].values
sle = s_pred[c['sle']].values
shc = s_pred[c["shc"]].values
she = s_pred[c["she"]].values
 
wt_pred = np.hstack((sp2wt(xna,sle),sp2wt(xna,shc),sp2wt(xna,she)))
wt_pred = pd.DataFrame(wt_pred,columns=wt_real.columns)
pd.options.display.max_rows = 999
result = part1.show_metrics(wt_real,wt_pred,e=0.01)
result
import joblib
joblib.dump(part1,'part1.pkl')
result.to_csv('result(162wt%)')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.18949133594186696, 'fraction_numerical': 0.025712688652878703, 'mean_word_length': 6.816593886462882, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '270332', 'license': 'MIT', 'n_tokens_mistral': 820, 'n_tokens_neox': 808, 'n_words': 120, 'path': 'examples/autorch_example_gpu.ipynb', 'repo_name': 'skywalker0803r/autorch', 'size': '1789'}