<jupyter_start><jupyter_text>Import libraries<jupyter_code>import pandas as pd
import numpy as np
import json
from IOHMM import UnSupervisedIOHMM
from IOHMM import OLS, DiscreteMNL, CrossEntropyMNL<jupyter_output><empty_output><jupyter_text>Read and clean data<jupyter_code>df = pd.read_csv('assistment_data_corrected.csv', encoding="ISO-8859-1")
df = df[df['overlap_time'] > 0]
df = df[df['ms_first_response'] > 0]<jupyter_output><empty_output><jupyter_text>Create model and set up parameters<jupyter_code>hmm = UnSupervisedIOHMM(num_states=2)
hmm.set_inputs(covariates_initial=[], covariates_transition=[], covariates_emissions=[['overlap_time']])
hmm.set_models(model_emissions = [OLS()], 
                model_transition=CrossEntropyMNL(solver='lbfgs'),
                model_initial=CrossEntropyMNL(solver='lbfgs'))
hmm.set_outputs([['correct']])
hmm.set_data([df])<jupyter_output><empty_output><jupyter_text>Train the model The below cell runs the EM algorithm for 100 iterations. This takes a long time (~1h)!<jupyter_code>hmm.train()<jupyter_output><empty_output><jupyter_text>Model parameters<jupyter_code># The coefficients of the OLS model for each hidden states
print(hmm.model_emissions[0][0].coef)
print(hmm.model_emissions[1][0].coef)
# The scale/dispersion of the OLS model for each hidden states
print(np.sqrt(hmm.model_emissions[0][0].dispersion))
print(np.sqrt(hmm.model_emissions[1][0].dispersion))
# The transition probability between two hidden states
print(np.exp(hmm.model_transition[0].predict_log_proba(np.array([[]]))))
print(np.exp(hmm.model_transition[1].predict_log_proba(np.array([[]]))))<jupyter_output>[[0.73395324 0.26604676]]
[[0.47911331 0.52088669]]<jupyter_text>Save model to file<jupyter_code>json_dict = hmm.to_json('./')
with open('./model.json', 'w') as outfile:
    json.dump(json_dict, outfile, indent=4, sort_keys=True)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13825857519788917, 'fraction_numerical': 0.031134564643799472, 'mean_word_length': 8.386138613861386, 'pattern_counts': {'":': 0, '<': 24, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '430220', 'license': 'MIT', 'n_tokens_mistral': 735, 'n_tokens_neox': 678, 'n_words': 115, 'path': 'notebooks/03 - IOHMM.ipynb', 'repo_name': 'aks96/student-modeling-project', 'size': '1895'}