<jupyter_start><jupyter_text>Load the data<jupyter_code>train_path = config.merged_path
filenames = os.listdir(train_path)
filenames = [fn for fn in filenames if os.path.isfile(os.path.join(train_path,fn))]
filenames = [os.path.join(train_path,filename) for filename in filenames]
print('Processing {} files'.format(len(filenames)))
datasets = [pd.read_csv(fn) for fn in filenames]
for i in range(len(datasets)):
    datasets[i] = datasets[i].set_index('timestamp')
    datasets[i].index = pd.to_datetime(datasets[i].index) 
# Take one example datset and have a look
example_index = 0
example_data = datasets[example_index]

example_data.head()<jupyter_output><empty_output><jupyter_text>Prepare data for HSMM<jupyter_code>X_list = [d[config.hsmmconfig.column_names].as_matrix() for d in datasets]<jupyter_output><empty_output><jupyter_text>Train HSMM<jupyter_code>#Note that with many iterations, the visualization becomes badly visible
model = hsmm.train_hsmm(X_list, Nmax=config.hsmmconfig.Nmax,
                                        nr_resamples=config.hsmmconfig.nr_resamples,
                                        save_model_path=config.model_path,
                                        trunc=config.hsmmconfig.truncate, visualize=True, verbose=True)<jupyter_output>Training iteration 0
Resampled 2 sequences in 4.7 seconds
Log likelihood:  42969.4489153
Convergence: average Hamming distance is 0.930594679186
Training iteration 1
Resampled 2 sequences in 4.4 seconds
Log likelihood:  54372.3138211
Convergence: average Hamming distance is 0.290258215962<jupyter_text>Plot one example<jupyter_code>import matplotlib.pyplot as plt
plt.subplots(figsize=(20, 5))
model.plot_stateseq(example_index)<jupyter_output><empty_output><jupyter_text>Save the data with the states Save the data including the states found. This labeled data serves as an input to the analyses.<jupyter_code>for i, dat in enumerate(datasets):
    dat['state'] = model.stateseqs[i]
    fn = str(str(dat['subset'][0]) + dat['filename'][0])+'.csv'
    dat.to_csv(os.path.join(config.states_path, fn))<jupyter_output><empty_output><jupyter_text>Save the model NB: This removes the data from the model! Only run this if you've saved the data with states.<jupyter_code>import pickle
#Remove the data from the model
model.states_list = []

with open(config.model_file, 'wb') as f:
    pickle.dump(model, file=f)<jupyter_output><empty_output><jupyter_text>Save the config<jupyter_code>from shutil import copyfile

copyfile('../config.yml', config.config_file)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10300429184549356, 'fraction_numerical': 0.02497073741708935, 'mean_word_length': 5.208232445520581, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 3, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '385331', 'license': 'Apache-2.0', 'n_tokens_mistral': 905, 'n_tokens_neox': 839, 'n_words': 193, 'path': 'python/notebooks/1_HSMM.ipynb', 'repo_name': 'wadpac/millenniumcohort-acc', 'size': '2563'}