<jupyter_start><jupyter_text>DMR-Gene CorrelationCalculate the partial correlation between DMR and Genes using clusters/samples as features.  Import<jupyter_code>import seaborn as sns
import matplotlib.pyplot as plt
from ALLCools.mcds import RegionDS
from ALLCools.mcds.correlation import region_correlation, get_corr_table<jupyter_output><empty_output><jupyter_text>Load Data<jupyter_code># DMR data
dmr = RegionDS.open('../RegionDS/test_HIP/')

# gene data
gene = RegionDS.open('test_HIP_Cluster/', region_dim='geneslop2k')
# global mC fractions, calculated from cell level MCDS by sum up cluster and genome base calls
sample_mch = gene.get_coords('sample_CHN')
sample_mcg = gene.get_coords('sample_CGN')<jupyter_output><empty_output><jupyter_text>DMR InputA 2-D sample-by-DMR DataArray, values are mCG fractions<jupyter_code>data_a = dmr['dmr_da_frac'].transpose('sample', 'dmr')
data_a<jupyter_output><empty_output><jupyter_text>Gene InputA 2-D sample-by-gene DataArray, values are cluster-normalized mCH fractions<jupyter_code>data_b = gene.sel(
    mc_type='CHN',
    sample=data_a.get_index('sample'))['geneslop2k_da_frac'].transpose(
        'sample', 'geneslop2k')
data_b<jupyter_output><empty_output><jupyter_text>Calculate correlations between DMR and genes<jupyter_code>true_results, null_results = region_correlation(data_a,
                                                data_b,
                                                sample_mch,
                                                sample_mcg,
                                                method='pearson',
                                                max_dist=1000000,
                                                cpu=40,
                                                null='sample',
                                                null_n=100000)
true_results.X.data.size, null_results.size<jupyter_output><empty_output><jupyter_text>Plot correlation distribution<jupyter_code>import numpy as np

fig, ax = plt.subplots(dpi=200)
sns.histplot(ax=ax,
             data=np.random.choice(null_results, 100000),
             label='Null',
             bins=100,
             color='grey')
sns.histplot(ax=ax,
             data=np.random.choice(true_results.X.data, 100000),
             label='True',
             bins=100,
             color='red')
ax.legend()<jupyter_output><empty_output><jupyter_text>Get Significantly Correlated Pairs<jupyter_code>corr_table = get_corr_table(true_results,
                            null_results,
                            region_dim_a=data_a.dims[1],
                            region_dim_b=data_b.dims[1],
                            direction='+',
                            alpha=0.03)

corr_table['pair_distance'] = corr_table['dmr_pos'] - corr_table['geneslop2k_pos']

corr_table<jupyter_output>Correlation cutoff corr > 0.5499999999999998
2855 correlation pairs selected.<jupyter_text>Plot distance and correlation distribution<jupyter_code>fig, axes = plt.subplots(figsize=(4, 4), nrows=2, dpi=200, constrained_layout=True, sharex=True)

ax = axes[0]
sns.scatterplot(ax=ax,
                data=corr_table,
                x='pair_distance',
                y='corr',
                s=0.2,
                linewidth=0)
ax = axes[1]
sns.histplot(corr_table['pair_distance'], ax=ax)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09793351302785265, 'fraction_numerical': 0.02365977837675951, 'mean_word_length': 2.4575569358178053, 'pattern_counts': {'":': 0, '<': 32, '<?xml version=': 0, '>': 33, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 16, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '8', 'dir': 'jupyter-structured-clean-dedup', 'id': '108704', 'license': 'MIT', 'n_tokens_mistral': 1110, 'n_tokens_neox': 1007, 'n_words': 167, 'path': 'docs/allcools/cluster_level/Correlation/02.dmr_gene_corr.ipynb', 'repo_name': 'jksr/ALLCools', 'size': '3339'}