<jupyter_start><jupyter_text>Import qbiome<jupyter_code>import qbiome
from qbiome.data_formatter import DataFormatter
from qbiome.quantizer import Quantizer
from qbiome.qnet_orchestrator import QnetOrchestrator
from qbiome.mask_checker import MaskChecker
from qbiome.forecaster import Forecaster
from qbiome.hypothesis import Hypothesis
from qbiome.qutil import customDataFormatter<jupyter_output><empty_output><jupyter_text>Import other packages<jupyter_code>import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set_theme()
plt.style.use('ggplot')<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>dirname = 'example_data/'
data = dirname + 'hc_Phylum_GTDB_denoised_filtered_percent.csv'
meta = dirname + 'hc_metadata.csv'<jupyter_output><empty_output><jupyter_text>Instantiate the data formatter, quantizer, qnet orchestrator<jupyter_code>BIOMES = ['Actinobacteriota', 'Bacteroidota', 'Firmicutes', 'Proteobacteria', 'unclassified_Bacteria']
data,property_map=customDataFormatter(data,meta,
                        META_PROP=['hc', 'HCZ', 'sex', 'hc_Birth',
           'HCZ_Birth', 'hc_Discharge', 'HCZ_Discharge',
           'HCZ_discharge_birth_difference', 'HCZ_Category',
           'HCZ_difference_Category', 'NEC', 'IVH', 'ROP', 'BPD', 'Infection',
           'total_Morbidities', 'Morbidity_Category', 'TIMPS score',
           'TIMPS z-score', 'TIMPS Category'],
                        COL_SELECT={"Samples": "sample_id", "PTID": "subject_id", "PMA":"week"},BIOMES=BIOMES)
quantizer = Quantizer()
qnet_orchestrator = QnetOrchestrator(quantizer)<jupyter_output><empty_output><jupyter_text>Load, quantize, and convert the data to qnet input format<jupyter_code>#data = formatter.load_data(data, meta)
quantized = quantizer.quantize_df(data)
features, label_matrix = quantizer.get_qnet_inputs(quantized)
quantized.head()
len(features)<jupyter_output><empty_output><jupyter_text>Train qnet<jupyter_code># should take < 5 min to run
qnet_orchestrator.train_qnet(
    features, label_matrix, alpha=0.2, min_samples_split=2, 
    out_fname='data1.joblib')<jupyter_output><empty_output><jupyter_text>Use the mask checker to mask 20% of the label matrix and use qnet to predict<jupyter_code>mask_checker = MaskChecker(qnet_orchestrator)
# takes 2 minutes to run
predicted = mask_checker.mask_and_predict(label_matrix, mask_percent=20)
predicted.head()<jupyter_output><empty_output><jupyter_text>Plot the predicted vs. original biome measurements<jupyter_code>BIOMES = ['Actinobacteriota', 'Bacteroidota', 'Firmicutes', 'Proteobacteria', 'unclassified_Bacteria']

concat = pd.concat([
    data.assign(source='original'), 
    predicted.assign(source='predicted')
])
concat = concat[concat.variable.isin(BIOMES)]

g = sns.FacetGrid(concat, col='variable', col_wrap=2, sharey=False, margin_titles=True)
g.map_dataframe(sns.lineplot, 'week', 'value', hue='source', ci=None, marker='o',
                    linewidth=2, alpha=0.75, markersize=5)
g.set_titles(row_template = '{row_name}', col_template = '{col_name}')
g.add_legend()<jupyter_output><empty_output><jupyter_text>It appears that the prediction passes our eyeball sanity check. We zoom in to look at the first 20 weeks.<jupyter_code>concat[(concat.subject_id==100001) & (concat.variable=='Actinobacteriota') & (concat.week.between(34,40))]
concat = concat[(concat.week <= 40)]

g = sns.FacetGrid(concat, col='variable', col_wrap=2, sharey=False, margin_titles=True)
g.map_dataframe(sns.lineplot, 'week', 'value', hue='source', ci=None, marker='o',
                    linewidth=2, alpha=0.75, markersize=5)
g.set_titles(row_template = '{row_name}', col_template = '{col_name}')
g.add_legend()<jupyter_output><empty_output><jupyter_text>To increase the precision of the prediction, we can apply the random forest regressor in the quantizer. See the example below when we use the forecaster.  Use the forecaster to sequentially mask part of the original data and predict<jupyter_code>forecaster = Forecaster(qnet_orchestrator)
# takes 10 minutes to run
# can specify an end week or default to the max end week in the data
forecasted = forecaster.forecast_data(
    label_matrix, 
    start_week=31,
#     end_week=data.week.max()
)
label_matrix<jupyter_output><empty_output><jupyter_text>Apply the random forest regressor in the quantizer<jupyter_code>dequantized = quantizer.dequantize_to_df(label_matrix)
dequantized_plot = quantizer.melt_into_plot_format(dequantized)
quantizer.fit_random_forest(data, dequantized_plot)
dequantized_plot
forecasted[(forecasted.subject_id==100001) & (forecasted.variable=='Actinobacteriota')]
data[(data.subject_id==100001) & (data.variable=='Actinobacteriota')].sort_values('week')
avg_forecasted_regressed = quantizer.apply_random_forest_regressor(
    forecasted#.fillna(method='ffill')
)
concat = pd.concat([
    data.assign(source='original'), 
    avg_forecasted_regressed.assign(source='forecasted')
])
concat = concat[concat.variable.isin(BIOMES)]
#concat = concat[(concat.week <= 20)]

g = sns.FacetGrid(concat, col='variable', col_wrap=2, sharey=False, margin_titles=True)
g.map_dataframe(sns.lineplot, 'week', 'value', hue='source', ci=None, marker='o',
                    linewidth=2, alpha=0.75, markersize=5)
g.set_titles(row_template = '{row_name}', col_template = '{col_name}')
g.add_legend()<jupyter_output><empty_output><jupyter_text>Use the forecaster to predict new dataWe construct a new patient from the quantized data column names because it contains all the feature names that the qnet uses.<jupyter_code>new_subject_id = '101'
np.random.seed(10001)
# initialize with random floats
new_patient = pd.DataFrame(
    np.random.rand(1, len(quantized.columns)),
    index=[0],
    columns=quantized.columns
)
new_patient.subject_id = new_subject_id
new_patient
new_patient_seq = new_patient.drop(columns='subject_id').to_numpy().squeeze()
new_patient_seq.shape
# suppose we only have 6 weeks of data for the new patient
# we mask out the remainder of the weeks
new_patient_masked = qnet_orchestrator.mask_sequence_at_weeks(
    new_patient_seq, 
    start_week=29, 
    end_week=qnet_orchestrator.get_max_timestamp(),
    fill_value=np.nan
)
new_patient_df = quantizer.add_meta_to_matrix(
    new_patient_masked[np.newaxis, :],
    add_subject_id=False
)
# manually add back subject_id
new_patient_df.insert(0, 'subject_id', [new_subject_id])
new_patient_df
new_patient_df = quantizer.melt_into_plot_format(new_patient_df)
new_patient_df
# quantize
new_patient_quantized = quantizer.quantize_df(new_patient_df)
# convert this patient into a format acceptable by the qnet
_, new_patient_matrix = quantizer.get_qnet_inputs(new_patient_quantized)
new_patient_matrix.shape<jupyter_output><empty_output><jupyter_text>Forecast the data starting from week 7 to the last week<jupyter_code># takes 2 minutes to run
new_patient_forecasted = forecaster.forecast_data(
    new_patient_matrix, 
    start_week=29
)
new_patient_forecasted.head()<jupyter_output><empty_output><jupyter_text>The NaNs are due to the random values we generated falling outside the quantization range.<jupyter_code>plottable = new_patient_forecasted[new_patient_forecasted.variable.isin(BIOMES)]

g = sns.FacetGrid(plottable, col='variable', col_wrap=2, sharey=False, margin_titles=True)
g.map_dataframe(sns.lineplot, 'week', 'value', ci=None, marker='o',
                    linewidth=2, alpha=0.75, markersize=5)
g.set_titles(row_template = '{row_name}', col_template = '{col_name}')
g.add_legend()
# zoom into the first 20 weeks
plottable = new_patient_forecasted[
    (new_patient_forecasted.variable.isin(BIOMES)) &
    (new_patient_forecasted.week <= 20)
     ]

g = sns.FacetGrid(plottable, col='variable', col_wrap=2, sharey=False, margin_titles=True)
g.map_dataframe(sns.lineplot, 'week', 'value', ci=None, marker='o',
                    linewidth=2, alpha=0.75, markersize=5)
g.set_titles(row_template = '{row_name}', col_template = '{col_name}')
g.add_legend()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11429282868525896, 'fraction_numerical': 0.011703187250996016, 'mean_word_length': 6.621442125237191, 'pattern_counts': {'":': 3, '<': 61, '<?xml version=': 0, '>': 57, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '317092', 'license': 'MIT', 'n_tokens_mistral': 2902, 'n_tokens_neox': 2763, 'n_words': 538, 'path': 'examples/old_notebooks/qbiome_example_data1.ipynb', 'repo_name': 'zeroknowledgediscovery/qbiome', 'size': '8032'}