<jupyter_start><jupyter_text>Set paths<jupyter_code>folder_path = abspath("./data/mcmicro_quants")
metadata_path = abspath("./data/metadata.csv")
image_raw_path = abspath("./data/cell2.tif")
image_mask_path = abspath("./data/cell2_mask.tif")
image_quants_path = abspath("./data/mcmicro_quants/mesmer-N-2016-2156-1_hBN220215_16738-35540_cell.csv")<jupyter_output><empty_output><jupyter_text>Set plot style<jupyter_code># Set palette
sns.set_style("darkgrid")
palette = {"MBM":"#A80D11",
           "ECM":"#008DB8"}<jupyter_output><empty_output><jupyter_text>Load metadata<jupyter_code># Read in metadata, set file name as index
metadata = pd.read_csv(metadata_path, index_col="file")
metadata.head()
%%time
# Read all marker intensities
marker = "CD68"
all_data = read_marker(folder_path, marker)
%%time
# Plot a distribution of the marker intensity across all images
plot_marker_intensity(all_data,  marker,
                      out=[abspath("./out/mets-density_CD68-raw.pdf"),
                           abspath("./out/mets-density_CD68-raw.png")])
# Trying to get a cutoff value for CD68 positive cells
cutoff = np.percentile(all_data, q=95)
cutoff
%%time
# show cutoff value
plot_marker_intensity(all_data, marker, cutoff,
                      out=[abspath("./out/mets-density_CD68-raw-gated.pdf"),
                           abspath("./out/mets-density_CD68-raw-gated.png")])
%%time
# Get log data and visualize
all_data_log = np.log(all_data)
all_data_log = all_data_log[~np.isinf(all_data_log)]
%%time
# Log data
plot_marker_intensity(all_data_log,  marker,
                      out=[abspath("./out/mets-density_CD68-log.pdf"),
                           abspath("./out/mets-density_CD68-log.png")])
plot_marker_intensity(all_data_log,  marker, np.log(cutoff),
                      out=[abspath("./out/mets-density_CD68-log-gated.pdf"),
                           abspath("./out/mets-density_CD68-log-gated.png")])<jupyter_output><empty_output><jupyter_text>Plot intensity distribution by tissue<jupyter_code>%%time
all_images = read_markers_labeled(folder_path, marker, metadata[["tissue", "sample"]])
all_images = all_images.sort_values(by="tissue", ascending=False)
all_images.head()
%%time
# Log scale the data
scaled = np.log(all_images[marker])
scaled = pd.DataFrame(scaled, columns=[marker])
scaled = scaled[~np.isinf(scaled[marker])]
scaled["tissue"] = all_images["tissue"]
scaled["sample"] = all_images["sample"]
scaled = scaled.sort_values(by="tissue", ascending=False)
scaled.head()
%%time
# Plot intensity distribution by tissue
plot_distributions(all_images, marker, hue="tissue", palette=palette,
                            out=[abspath("./out/mets-violin_cd68-color_MBM_ECM-raw.pdf"),
                                 abspath("./out/mets-violin_cd68-color_MBM_ECM-raw.png")])
%%time
# Plot intensity distribution by tissue
plot_distributions(scaled, marker, hue="tissue", palette=palette,
                            out=[abspath("./out/mets-violin_cd68-color_MBM_ECM-log.pdf"),
                                 abspath("./out/mets-violin_cd68-color_MBM_ECM-log.png")])<jupyter_output>CPU times: total: 55.2 s
Wall time: 55.4 s<jupyter_text>Visualize and check if threshold makes sense<jupyter_code>%%time
# Visualize threshold
plot_centroids(image_quants_path, marker, cutoff,
               out=abspath(f"./out/{basename(image_raw_path)}.html"))<jupyter_output>CPU times: total: 4.11 s
Wall time: 1.26 s<jupyter_text>Get CD68 mean marker intensities<jupyter_code>%%time
metadata["CD68_intensity"] = get_marker_intensities(folder_path, marker, cutoff=0)
metadata.head()<jupyter_output>CPU times: total: 37.8 s
Wall time: 5.18 s<jupyter_text>Get CD68+ mean intensities<jupyter_code>%%time
metadata["CD68+ Intensity"] = get_marker_intensities(folder_path, marker, cutoff=cutoff)
metadata.head()<jupyter_output>CPU times: total: 36 s
Wall time: 4.74 s<jupyter_text>Get marker positive cells<jupyter_code>%%time
metadata["CD68_cells"] = get_marker_positive_cells(folder_path, marker, cutoff)
metadata.head()<jupyter_output>CPU times: total: 36.2 s
Wall time: 5.42 s<jupyter_text>Get proportion of marker positive cells<jupyter_code>%%time
metadata["CD68_cell_proportion"] = metadata["CD68_cells"]/metadata["n"]
metadata.head()<jupyter_output>CPU times: total: 0 ns
Wall time: 1.1 ms<jupyter_text>Plot Marker Intensity for CD68<jupyter_code>%%time
plot_distribution_boxen(metadata, "CD68_intensity", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-cd68_mean_intensity.pdf"),
                             abspath("./out/mets-boxen-cd68_mean_intensity.png")])
%%time
plot_sample_distributions(metadata, x="sample", marker="CD68_intensity", hue="tissue", palette=palette,
                          out=[abspath("./out/mets-scatter-cd68_mean_intensity_per_sample.pdf"),
                               abspath("./out/mets-scatter-cd68_mean_intensity_per_sample.png")])<jupyter_output>CPU times: total: 1.86 s
Wall time: 2.04 s<jupyter_text>Plot Marker Intensity for CD68+ cells<jupyter_code>%%time
plot_distribution_boxen(metadata, "CD68+ Intensity", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-CD68pos_mean_intensity.pdf"),
                             abspath("./out/mets-boxen-CD68pos_mean_intensity.png")])
plot_sample_distributions(metadata, x="sample", marker="CD68+ Intensity", hue="tissue", palette=palette,
                          out=[abspath("./out/mets-scatter-CD68pos_mean_intensity_per_sample.pdf"),
                               abspath("./out/mets-scatter-CD68pos_mean_intensity_per_sample.png")])<jupyter_output><empty_output><jupyter_text>Number of CD68 positive cells<jupyter_code>plot_distribution_boxen(metadata, marker="CD68_cells", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-cd68_cells.pdf"),
                             abspath("./out/mets-boxen-cd68_cells.png")])
%%time
# Plot CD68  positve cells per sample
plot_sample_distributions(metadata, x="sample", marker="CD68_cells", hue="tissue", palette=palette,
                          out=[abspath("./out/mets-scatter-cd68_cells_per_sample.pdf"),
                               abspath("./out/mets-scatter-cd68_cells_per_sample.png")])<jupyter_output>CPU times: total: 1.95 s
Wall time: 2.05 s<jupyter_text>Proportion of CD68 positive cells<jupyter_code>%%time
# Plot CD68 positve cells per tissue
plot_distribution_boxen(metadata, marker="CD68_cell_proportion", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-cd68_proportion.pdf"),
                             abspath("./out/mets-boxen-cd68_proportion.png")])
%%time
# Plot CD68 cell proportion per sample
plot_sample_distributions(metadata, x="sample", marker="CD68_cell_proportion", hue="tissue", palette=palette,
                          out=[abspath("./out/mets-scatter-cd68_cells_proportion_per_sample.pdf"),
                               abspath("./out/mets-scatter-cd68_cells_proportion_per_sample.png")])<jupyter_output>CPU times: total: 1.91 s
Wall time: 1.99 s<jupyter_text>Create interative figure<jupyter_code>%%time
# Read in raw image and mask
image_raw = io.imread(image_raw_path)
image_mask = io.imread(image_mask_path)
image_quants = dt.fread(image_quants_path, columns={"CellID":"CellID", marker:marker, "X_centroid":"X", "Y_centroid":"Y", ...:None}).to_pandas()

plot_cell_contours_plotly(image_raw, image_mask, image_quants, out=abspath("../out/cd68_mask_test.html"), marker=marker, cutoff=cutoff, channel=2)<jupyter_output>CPU times: total: 13min 59s
Wall time: 14min 3s<jupyter_text>Stats for number of cells<jupyter_code># Wilcoxon-Ranksum test to check the difference between the amount of cells on MBM vs ECM tissue
res = ranksums(metadata[metadata["tissue"]=="MBM"]["CD68_cells"], metadata[metadata["tissue"]=="ECM"]["CD68_cells"])
res
for g, d in metadata.groupby("tissue"):
    print(f"Mean {marker} positve cells per image in {g} = {d['CD68_cells'].mean()}")
    print(f"STD {marker} positve cells per image in {g} = {d['CD68_cells'].mean()}")<jupyter_output>Mean CD68 positve cells per image in ECM = 747.2676056338029
STD CD68 positve cells per image in ECM = 747.2676056338029
Mean CD68 positve cells per image in MBM = 1766.6943005181347
STD CD68 positve cells per image in MBM = 1766.6943005181347<jupyter_text>Stats for cell proportion<jupyter_code># Wilcoxon-Ranksum test to check the difference between the proportion of cells on MBM vs ECM tissue
res_prop =  ranksums(metadata[metadata["tissue"]=="MBM"]["CD68_cell_proportion"], metadata[metadata["tissue"]=="ECM"]["CD68_cell_proportion"])
res_prop
# Get mean and std  by tissue for CD68 cell proportion
for g, d in metadata.groupby("tissue"):
    print(f"Mean {marker} cell proportion per image in {g} = {d['CD68_cell_proportion'].mean():.4f}")
    print(f"STD {marker} cell proportion per image in {g} = {d['CD68_cell_proportion'].mean():.4f}")<jupyter_output>Mean CD68 cell proportion per image in ECM = 0.0228
STD CD68 cell proportion per image in ECM = 0.0228
Mean CD68 cell proportion per image in MBM = 0.1090
STD CD68 cell proportion per image in MBM = 0.1090<jupyter_text>Stats for Intensity  Wilcoxon ransum test for raw intensity<jupyter_code>%%time
res_int = ranksums(all_images[all_images["tissue"]=="MBM"][marker], all_images[all_images["tissue"]=="ECM"][marker])
res_int
for g, d in all_images.groupby("tissue"):
    print(f"Mean intensity of {marker} cells in {g} = {d[marker].mean():.4f}")
    print(f"STD  intensity of {marker} cells in {g} = {d[marker].std():.4f}")
CD68pos = all_images[all_images[marker] >= cutoff]
res_int_CD68pos = ranksums(CD68pos[CD68pos["tissue"]=="MBM"][marker], CD68pos[CD68pos["tissue"]=="ECM"][marker])
res_int_CD68pos
# Calculate fold change
np.log2(0.7442/0.3195)<jupyter_output><empty_output><jupyter_text>Wilcoxon ranksum test for mean intensity on CD68 marker<jupyter_code>%%time
res_mean_int = ranksums(metadata[metadata["tissue"]=="MBM"]["CD68_intensity"], metadata[metadata["tissue"]=="ECM"]["CD68_intensity"])
res_mean_int<jupyter_output>CPU times: total: 15.6 ms
Wall time: 4.99 ms<jupyter_text>Wilcoxon ranksum test for mean intensity on CD68+ cells<jupyter_code>res_mean_int_CD68pos = ranksums(metadata[metadata["tissue"]=="MBM"]["CD68+ Intensity"], metadata[metadata["tissue"]=="ECM"]["CD68+ Intensity"])
res_mean_int_CD68pos<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13702707893220664, 'fraction_numerical': 0.035817169195314, 'mean_word_length': 4.738292011019284, 'pattern_counts': {'":': 5, '<': 64, '<?xml version=': 0, '>': 65, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '19', 'dir': 'jupyter-structured-clean-dedup', 'id': '56445', 'license': 'MIT', 'n_tokens_mistral': 3966, 'n_tokens_neox': 3633, 'n_words': 652, 'path': 'multiplexed_image_analysis/notebook/cd68.ipynb', 'repo_name': 'IzarLab/Melanoma_Brain_Metastasis', 'size': '10414'}