<jupyter_start><jupyter_text>KISS ModelSimplest possible model that still (sort of) accomplishes our goal, mostly for rapid prototyping<jupyter_code>import onnxruntime as rt
import numpy as np
from sklearn.model_selection import train_test_split
import pandas as pd
import matplotlib.pyplot as plt
from sklearn.decomposition import TruncatedSVD
from sklearn.feature_extraction.text import TfidfVectorizer
from sklearn.ensemble import RandomForestClassifier
from sklearn.pipeline import make_pipeline
from sklearn.pipeline import Pipeline
from skl2onnx import convert_sklearn
from skl2onnx.common.data_types import StringTensorType
from sklearn.linear_model import LogisticRegression
from sklearn.compose import ColumnTransformer

from nltk.corpus import stopwords

from sklearn.dummy import DummyClassifier
from sklearn.metrics import accuracy_score, fbeta_score, jaccard_score
diagnosis_df = pd.read_csv(
    "../data/mimiciii-14/DIAGNOSES_ICD.csv.gz", usecols=["HADM_ID", "ICD9_CODE"]
)
note_events_df = pd.read_csv(
    "../data/mimiciii-14/NOTEEVENTS.csv.gz", usecols=["HADM_ID", "TEXT"]
)
df = (note_events_df.merge(diagnosis_df)
      .set_index("HADM_ID")
      .dropna()
      .reset_index())
df.head()
df = df.reset_index()
codes_freq = df["ICD9_CODE"].value_counts()
most_frequent = set(codes_freq.index[:10])
df_subset = df[df.ICD9_CODE.isin(most_frequent)]
df_subset.shape
df_subset.ICD9_CODE.unique()
df_subset2 = df_subset.sample(n=50000)
df_subset2.shape
icd9_codes = list(df_subset2.ICD9_CODE.unique())
ddf = pd.DataFrame(index=df_subset2.HADM_ID.unique(), columns=["TEXT"] + icd9_codes).fillna(False)
for idx, row in df_subset2.iterrows():
    ddf.loc[row.HADM_ID, "TEXT"] = row.TEXT
    ddf.loc[row.HADM_ID, row.ICD9_CODE] = True
ddf.head()
ddf[icd9_codes].sum(axis=1).max()
X = ddf[["TEXT"]].values
y = ddf[icd9_codes[0]].values.astype("int64")
X.shape, y.shape
X_train, X_test, y_train, y_test = \
    train_test_split(X, y, test_size=0.33, random_state=42)
pipeline = Pipeline(steps=[
    ("reshape", ColumnTransformer([
        ("vectorize", TfidfVectorizer(), 0),
    ])),
    ("dimensionality_reduction", TruncatedSVD(n_components=50)),
    ("classifier", RandomForestClassifier()),
])
X_train.shape, y_train.shape
pipeline.fit(X_train, y_train)
# dummy = Pipeline(steps=[
#     ("reshape", ColumnTransformer([
#         ("vectorize", TfidfVectorizer(stop_words="english"), 0),
#     ])),
#     ("dimensionality_reduction", TruncatedSVD(n_components=50)),
#     ("classifier", DummyClassifier(strategy="uniform")),
# ])
# dummy.fit(X_train, y_train)
accuracy_score(y_test, dummy.predict(X_test)), accuracy_score(y_test, pipeline.predict(X_test))
jaccard_score(y_test, dummy.predict(X_test), average="weighted"), jaccard_score(y_test, pipeline.predict(X_test), average="weighted")
fbeta_score(y_test, dummy.predict(X_test), beta=1, average="weighted"), fbeta_score(y_test, pipeline.predict(X_test), beta=1, average="weighted")
pipeline.predict(X_test.iloc[0:2])
onx = convert_sklearn(
    pipeline,
    "autoicd-kiss",
    initial_types=[
        ("input", StringTensorType(shape=[None, 1]))
    ],
    options={
        TfidfVectorizer: {
            "separators": [
                ' ', '.', '\\?', ',', ';', ':', '!',
                '\\(', '\\)', '\n', '"', "'",
                "-", "\\[", "\\]", "@"
            ]
        }
    }
)
with open("rf_iris.onnx", "wb") as f:
    f.write(onx.SerializeToString())
sess = rt.InferenceSession("rf_iris.onnx")
pred_onx = sess.run(None, {"input": X_test})[0]
print("predict", pred_onx[0])
print("predict_proba", pred_onx[1])<jupyter_output>predict 1
predict_proba 1
{'fraction_non_alphanumeric': 0.15249105422515827, 'fraction_numerical': 0.014588494357280485, 'mean_word_length': 5.076923076923077, 'pattern_counts': {'":': 2, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '292571', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 1371, 'n_tokens_neox': 1336, 'n_words': 224, 'path': 'nbs/KISS-model.ipynb', 'repo_name': 'simonlevine/x-transformer-icd', 'size': '3633'}