<jupyter_start><jupyter_text>General helper functions  Making nice tables<jupyter_code>import tabulate
import IPython

class OrgFormatter(IPython.core.formatters.BaseFormatter):
    format_type = IPython.core.formatters.Unicode('text/org')
    print_method = IPython.core.formatters.ObjectName('_repr_org_')

def pd_dataframe_to_org(df):
    return tabulate.tabulate(df, headers='keys', tablefmt='orgtbl', showindex='always')

ip = get_ipython()
ip.display_formatter.formatters['text/org'] = OrgFormatter()

f = ip.display_formatter.formatters['text/org']
f.for_type_by_name('pandas.core.frame', 'DataFrame', pd_dataframe_to_org)<jupyter_output><empty_output><jupyter_text>Excel reader and loading count data<jupyter_code>import pandas as pd
import warnings
warnings.filterwarnings('ignore')


def read_xl(fn="/Users/aoife/PHD/Transcripts/Data/diff_from_col0:False_onlyDiff:False.xlsx", unique=True):
    xl = pd.ExcelFile(fn)
    sheet_names = xl.sheet_names
    dfs = []
    for s in sheet_names:
        d = xl.parse(s)
        if unique:
            d['sample'] = s.split("|")[0].replace(" ", "")
        else:
            d['sample'] = s
        dfs.append(d)

    DE = pd.concat(dfs)
    DE = DE.rename_axis('gene').sort_values(by=['gene', 'log2FoldChange'],
                                            ascending=[False, False])
    return DE

counts = pd.read_csv(
    "/Users/hughesn/PHD/Transcripts/Data/norml_count_data.csv",index_col=0)
counts[[c for c in counts.columns if 'cer_c' in c]].head(5)<jupyter_output><empty_output><jupyter_text>Gprofiler function<jupyter_code>from gprofiler import GProfiler

def get_gene_names(geneList):

    gp = GProfiler(return_dataframe=True)
    df = gp.convert(organism='athaliana',
               query=geneList)[['incoming', 'name', 'description']]
    df['description'] = df.apply(lambda x: x['description'].split('[')[0].split(';')[0], axis=1)
    return df
get_gene_names(list(counts.head(5).index))<jupyter_output><empty_output><jupyter_text>Misc plotting functions<jupyter_code>def get_locs(DE, n, include_large=True, include_small=True):
    samples = DE['sample'].unique()
    locs = []
    for idx, s in enumerate(samples):
        if include_large:
            l1 = DE[DE['sample'] == s][['log2FoldChange']].groupby(['gene']).sum().sort_values(by='log2FoldChange',
                                                                                               ascending=False).head(n).index.values
            locs.extend(l1)
        if include_small:
            l2 = DE[DE['sample'] == s][['log2FoldChange']].groupby(['gene']).sum().sort_values(by='log2FoldChange',
                                                                                               ascending=True).head(n).index.values
            locs.extend(l2)
    locs = np.array(locs)
    return locs


def make_clustermap_df_n_samples(DE, description='description', n=20, include_large=True, include_small=True):
    locs = get_locs(DE, n, include_large=include_large, include_small=include_small)
    top = DE.loc[locs]
    top = top.groupby(['sample', 'gene']).mean(
    ).reset_index().set_index('gene')
    top = top.pivot(columns='sample', values='log2FoldChange')
    top['gene name'] = list(get_gene_names(
        list(top.index.values))[description])
    # top = top.set_index('gene name')
    return top


def get_two_clustermaps_data(df, n):
    top = make_clustermap_df_n_samples(df, n=n, include_small=False)
    bot = make_clustermap_df_n_samples(df, n=n, include_large=False)
    return (top, bot)

def make_two_clustermaps(df, n=50):
    top, bot = get_two_clustermaps_data(df, n)
    gt = sns.clustermap(top, cmap='bwr', vmin=-10, vmax=10, yticklabels=True, figsize=(15,15))
    gb = sns.clustermap(bot, cmap='bwr', vmin=-10, vmax=10, yticklabels=True, figsize=(15,15))
    return (top, bot, gt, gb)


def make_clustermap_table(df, n=50):
    top = make_clustermap_df_n_samples(df, n=n, include_small=False)
    bot = make_clustermap_df_n_samples(df, n=n, include_large=False)
    return pd.concat([top, bot])<jupyter_output><empty_output><jupyter_text>Count Data  Count data feature ranking So it's worth noting that all expression profiles look similar, so choosing from 20k+ genes for being a key trait is not going to work.<jupyter_code>import pandas as pd
import matplotlib.pyplot as plt
import seaborn as sns
sns.set()
counts = pd.read_csv(
    "/Users/hughesn/PHD/Transcripts/Data/norml_count_data.csv", index_col=0)

filter_cols = [c for c in counts.columns if (
    c.startswith('col_w_6h') or c.startswith('col_c_6h'))]

col_counts = counts[filter_cols]

col_w_counts = col_counts[[c for c in col_counts.columns if '_w_' in c]]
col_c_counts = col_counts[[c for c in col_counts.columns if '_c_' in c]]

fig, ax = plt.subplots(1,3, sharex=True, sharey=True, figsize=(15,5))
axes = iter(ax)
for c,w in zip(col_c_counts.columns, col_w_counts.columns):
    a = next(axes)
    a.hist(col_c_counts[c], alpha=0.3)
    a.hist(col_w_counts[w], alpha=0.3)
    a.set_title(w.replace('_w_','_'))<jupyter_output><empty_output><jupyter_text>Specifically grouped genes to test with random forest  Grouped by counts *N.B. doesn't work well*<jupyter_code>import jenkspy
from sklearn.ensemble import RandomForestClassifier
from sklearn.model_selection import train_test_split
import numpy as np
import time

col0 = counts[[c for c in counts.columns if c.startswith('col_w_05h')]].mean(axis=1).sort_values()
col0A = np.array(col0)
breaks = jenkspy.jenks_breaks(col0A, nb_class=20)

col0_index = pd.Index(col0)

indices = []
for b in breaks:
    r = col0_index.get_loc(b)
    if isinstance(r, slice):
        r = r.stop
    indices.append(r)

group_locs = [df.index for df in np.split(col0, indices)]

cols = {}
for c in counts.columns:
    avg_vals = []
    for g in group_locs:
        avg_vals.append(np.mean(counts[c].loc[g]))
    cols[c] = avg_vals
refined_counts = pd.DataFrame(cols)
refined_counts = refined_counts.set_index(pd.Index(["group {0}".format(i) for i in refined_counts.index]))



subset = refined_counts[[c for c in refined_counts.columns if (c.startswith('col_w_05h') or c.startswith('col_c_05h')) ]]
data = refined_counts.T
data = data.reset_index()
data['index'] = data['index'].apply(lambda x: x.rsplit('_',1)[0])
data = data.set_index('index')
feat_labels = data.columns

t_start = time.time()
X_train, X_test, y_train, y_test = train_test_split(data.values.T, feat_labels, test_size=0.33, random_state=42)
forest = RandomForestClassifier(n_estimators=10000, random_state=0, n_jobs=-1)
forest.fit(X_train, y_train)
t_end = time.time()
importances = forest.feature_importances_
indices = np.argsort(importances)[::-1]

for feature in sorted(zip(feat_labels, forest.feature_importances_), key=lambda x: x[1]):
    print(feature)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13703051643192488, 'fraction_numerical': 0.012910798122065728, 'mean_word_length': 4.156580937972769, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 10, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '6', 'dir': 'jupyter-structured-clean-dedup', 'id': '29896', 'license': 'MIT', 'n_tokens_mistral': 2554, 'n_tokens_neox': 2476, 'n_words': 458, 'path': 'workbooks/organised/.ipynb_checkpoints/setup-checkpoint.ipynb', 'repo_name': 'SirSharpest/RNA-Seq-Analysis', 'size': '6816'}