<jupyter_start><jupyter_text>Load Data<jupyter_code>train_methyl_array=MethylationArray.from_pickle("train_val_test_sets/train_methyl_array.pkl")
val_methyl_array=MethylationArray.from_pickle("train_val_test_sets/val_methyl_array.pkl")
test_methyl_array=MethylationArray.from_pickle("train_val_test_sets/test_methyl_array.pkl")<jupyter_output><empty_output><jupyter_text>UMAP Embed<jupyter_code>umap = UMAP(n_components=100)
umap.fit(train_methyl_array.beta)
train_methyl_array.beta = pd.DataFrame(umap.transform(train_methyl_array.beta.values),index=train_methyl_array.return_idx())
val_methyl_array.beta = pd.DataFrame(umap.transform(val_methyl_array.beta),index=val_methyl_array.return_idx())
test_methyl_array.beta = pd.DataFrame(umap.transform(test_methyl_array.beta),index=test_methyl_array.return_idx())<jupyter_output>/anaconda2/envs/py36/lib/python3.6/site-packages/sklearn/metrics/pairwise.py:259: RuntimeWarning: invalid value encountered in sqrt
  return distances if squared else np.sqrt(distances, out=distances)<jupyter_text>Cluster Training Data<jupyter_code>def reduce_plot(data, labels, legend_title):
    np.random.seed(42)
    plt.figure(figsize=(8,8))
    t_data=pd.DataFrame(PCA(n_components=2).fit_transform(data),columns=['z1','z2'])
    t_data[legend_title]=labels
    sns.scatterplot('z1','z2',hue=legend_title, cmap=cubehelix_palette(as_cmap=True),data=t_data)

model = HDBSCAN(algorithm='best')
train_predicted_clusters = model.fit_predict(train_methyl_array.beta.astype(np.float64))
reduce_plot(train_methyl_array.beta, train_methyl_array.pheno['Age'].values,'Age')
reduce_plot(train_methyl_array.beta, train_predicted_clusters,'Cluster')
import tabulate
train_methyl_array.pheno['Cluster']=train_predicted_clusters
output_data=train_methyl_array.pheno.groupby('Cluster')['Age'].agg([np.mean,len])
print(tabulate.tabulate(output_data,headers='keys',tablefmt="pipe"))<jupyter_output>|   Cluster |    mean |   len |
|----------:|--------:|------:|
|        -1 | 50.8971 |    68 |
|         0 | 43.5    |    10 |
|         1 | 55.3529 |    17 |
|         2 | 56      |     8 |
|         3 | 72.4286 |     7 |
|         4 | 55.375  |     8 |
|         5 | 58.75   |     8 |
|         6 | 59.2353 |    17 |
|         7 | 36.3333 |     6 |
|         8 | 25.4074 |    27 |<jupyter_text>Supervised Model<jupyter_code>from sklearn.ensemble import RandomForestRegressor
from sklearn.metrics import r2_score

y_pred = {}
scores={}
model=MachineLearning(RandomForestRegressor,options={})
model.fit(train_methyl_array,val_methyl_array,'Age')
y_pred['train']=model.predict(train_methyl_array)
y_pred['val']=model.predict(val_methyl_array)
y_pred['test']=model.predict(test_methyl_array)

scores['train']=r2_score(train_methyl_array.pheno['Age'],y_pred['train'])
scores['val']=r2_score(val_methyl_array.pheno['Age'],y_pred['val'])
scores['test']=r2_score(test_methyl_array.pheno['Age'],y_pred['test'])
scores
data=pd.DataFrame(model.model.feature_importances_,columns=['Importance'])
data=data.sort_values('Importance').iloc[::-1]
data['sample']=np.arange(len(data.index))

plt.figure(figsize=(5,5))
sns.barplot('sample','Importance',data=data)
plt.axis('off')<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15324594257178528, 'fraction_numerical': 0.03245942571785269, 'mean_word_length': 6.418981481481482, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '530875', 'license': 'MIT', 'n_tokens_mistral': 1322, 'n_tokens_neox': 1180, 'n_words': 109, 'path': 'example_scripts/machine_learning_example/MachineLearningExample.ipynb', 'repo_name': 'holonomicjl/PyMethylProcess', 'size': '3204'}