<jupyter_start><jupyter_text>Exploring Batch 3 Replicate Correlation**<NAME>, 2019**<jupyter_code>import os
import numpy as np
import pandas as pd
import plotnine as gg

from pycytominer.cyto_utils.features import drop_outlier_features
from pycytominer import audit
def plot_replicate_corr(x_df):
    plot_obj = gg.ggplot(x_df, gg.aes(x="correlation", fill="replicate")) + \
        gg.geom_density(alpha = 0.3) + \
        gg.scale_fill_manual(name="Replicate",
                             labels={"True": "True",
                                     "False": "False"},
                             values=["#B99638", "#2DB898"]) + \
        gg.xlab("Pearson Correlation") + \
        gg.ylab("Density") + \
        gg.theme_light()
    return plot_obj

def plot_individual_replicate_corr(x_df):
    # Process input data to ensure all sample correlations are captured in
    # one column, but that replicate correlation is not captured
    left_df = (
        x_df
        .loc[:, ["Metadata_clone_number_b", "correlation", "replicate"]]
        .reset_index(drop=True)
        .rename({"Metadata_clone_number_b": "Metadata_clone_number"}, axis='columns')
    )
    right_df = (
        x_df
        .loc[:, ["Metadata_clone_number", "correlation", "replicate"]]
        .query("replicate == False")
        .reset_index(drop=True)
    )

    x_df = pd.concat([left_df, right_df], axis='rows', ignore_index=True)

    # Now plot
    plot_obj = gg.ggplot(x_df, gg.aes(y="correlation", x="replicate", fill="replicate")) + \
        gg.geom_jitter(shape = ".", size=0.5, alpha=0.3) + \
        gg.geom_boxplot(alpha=0.3, outlier_alpha=0) + \
        gg.scale_fill_manual(name="Replicate",
                             labels={"True": "True",
                                     "False": "False"},
                             values=["#B99638", "#2DB898"]) + \
        gg.xlab("Replicates") + \
        gg.ylab("Pearson Correlation") + \
        gg.theme_light() + \
        gg.theme(subplots_adjust={'wspace': 0.2},
                 axis_text=gg.element_text(size=7),
                 axis_title=gg.element_text(size=9),
                 strip_text=gg.element_text(size=6, color="black"),
                 strip_background=gg.element_rect(colour="black",
                                                  fill="#fdfff4")) + \
        gg.facet_wrap("~Metadata_clone_number")
    return plot_obj
batch = "2019_11_19_Batch5"
profile_dir = os.path.join("..", "..", "1.process-profiles", "profiles")
batch_dir = os.path.join(profile_dir, batch)

backend_folders = os.listdir(batch_dir)
plate_files = [
    os.path.join(batch_dir, x, "{}_normalized_variable_selected.csv".format(x))
    for x in backend_folders if ".DS_Store" not in x
]

print(plate_files)<jupyter_output>['../1.process-profiles/profiles/2019_11_19_Batch5/217755/217755_normalized_variable_selected.csv']<jupyter_text>Read in Files<jupyter_code>df = pd.read_csv(plate_files[0])
exclude_features = drop_outlier_features(df)
df = df.drop(exclude_features, axis="columns")

print(df.shape)
df.head(2)<jupyter_output>(60, 156)<jupyter_text>How Many Replicates?<jupyter_code>df.groupby(replicate_columns).size().reset_index().rename(columns={0:'replicate_count'})<jupyter_output><empty_output><jupyter_text>Get Pairwise Correlations<jupyter_code>mut_cor_df = audit(df, audit_groups=replicate_columns, audit_resolution="full")
for col in replicate_columns:
    a = mut_cor_df.columns.str.contains(col)
mut_cor_df.loc[:, mut_cor_df.columns[a]].head().difference()
replicate_columns
mut_cor_df = (
    mut_cor_df
    .assign(replicate = 
            (
                (
                    mut_cor_df.Metadata_clone_number_pair_a ==
                    mut_cor_df.Metadata_clone_number_pair_b
                ) &
                (
                    mut_cor_df.Metadata_treatment_pair_a ==
                    mut_cor_df.Metadata_treatment_pair_b
                )
            )
           )
)

print(mut_cor_df.shape)
mut_cor_df.head(2)
gg.ggplot(mut_cor_df, gg.aes(y="pairwise_correlation", x="replicate", fill="replicate")) + \
        gg.geom_jitter(shape = ".", size=0.5, alpha=0.3) + \
        gg.geom_boxplot(alpha=0.3, outlier_alpha=0) + \
        gg.scale_fill_manual(name="Replicate",
                             labels={"True": "True",
                                     "False": "False"},
                             values=["#B99638", "#2DB898"]) + \
        gg.xlab("Replicates") + \
        gg.ylab("Pearson Correlation") + \
        gg.theme_light() + \
        gg.theme(subplots_adjust={'wspace': 0.2},
                 axis_text=gg.element_text(size=7),
                 axis_title=gg.element_text(size=9),
                 strip_text=gg.element_text(size=6, color="black"),
                 strip_background=gg.element_rect(colour="black",
                                                  fill="#fdfff4")) + \
        gg.facet_grid("Metadata_treatment_pair_a~Metadata_clone_number_pair_a")<jupyter_output><empty_output><jupyter_text>Output Median Replicate CorrelationWe use this later for platemap visualization.<jupyter_code>wt_median_cor_df = (
    wt_cor_df
    .groupby(["Metadata_clone_number", "replicate"])
    .median()
    .reset_index()
)

mut_median_cor_df = (
    mut_cor_df
    .groupby(["Metadata_clone_number", "replicate"])
    .median()
    .reset_index()
)

# Save Files
file = os.path.join("results", "wed_median_replicate_correlation.tsv")
wt_median_cor_df.to_csv(file, sep='\t', index=False)

file = os.path.join("results", "th_median_replicate_correlation.tsv")
mut_median_cor_df.to_csv(file, sep='\t', index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13580028328611898, 'fraction_numerical': 0.01788243626062323, 'mean_word_length': 2.684931506849315, 'pattern_counts': {'":': 7, '<': 20, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '288836', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 2005, 'n_tokens_neox': 1860, 'n_words': 252, 'path': '1.profiling-audit/notebooks/replicate-correlation-example.ipynb', 'repo_name': 'DavidStirling/profiling-resistance-mechanisms', 'size': '5653'}