<jupyter_start><jupyter_text>Set paths<jupyter_code># Pathways to metadata and data
folder_path = abspath("./data/mcmicro_quants")
metadata_path = abspath("./data/metadata.csv")
image_raw_path = abspath("./data/cell.tif")
image_mask_path = abspath("./data/cell_mask.tif")
image_quants_path = abspath("./data/mcmicro_quants/mesmer-N-2016-2156-1_hBN220215_16738-35540_cell.csv")<jupyter_output><empty_output><jupyter_text>Set plot style<jupyter_code># Set palette
sns.set_style("darkgrid")
palette = {"MBM":"#A80D11",
           "ECM":"#008DB8"}<jupyter_output><empty_output><jupyter_text>Load metadata<jupyter_code># Read in metadata, set file name as index
metadata = pd.read_csv(metadata_path, index_col="file")
metadata.head()
# Get total number of identified cells
metadata["n"].sum()
#Get number of cells per tissue
for g, d in metadata.groupby("tissue"):
    print(f"#  Cells in {g} = {d['n'].sum()}")
    print(f"Fraction in {g} = {d['n'].sum()/metadata['n'].sum():.4f}")<jupyter_output>#  Cells in ECM = 5352494
Fraction in ECM = 0.5986
#  Cells in MBM = 3589181
Fraction in MBM = 0.4014<jupyter_text>Get a cutoff for CD138 positive cells<jupyter_code>%%time
# Iterate over all images and get only the values for CD138
# Read all marker intensities
marker = "CD138"
all_data = read_marker(folder_path, marker)
all_data.head()
# Get cutoff value based on all images
cutoff = np.percentile(all_data["CD138"], q=99)
print(cutoff)
# show cutoff value
plot_marker_intensity(all_data, marker, cutoff,
                      out=[abspath("./out/mets-density_CD138-raw-gated.pdf"),
                           abspath("./out/mets-density_CD138-raw-gated.png")])<jupyter_output><empty_output><jupyter_text>Check if threshold makes sense<jupyter_code># Visualize threshold
plot_centroids(image_quants_path, marker, cutoff,
               out=abspath(f"./out/{basename(image_raw_path)}_CD138.html"))<jupyter_output><empty_output><jupyter_text>Calculate the average euclidean distance for CD138 positive cells per image<jupyter_code>%%time
metadata["distances"] = measure_marker_positive_mean_distance(folder_path, marker, cutoff)
metadata.head()
# Images without CD138 positive cells
metadata["distances"].isna().sum()
# CD138 average distance
sns.set_style("darkgrid")
fig, ax = plt.subplots(1, 1, figsize=(10, 10))
sns.boxenplot(data=metadata, x="tissue", y="distances", hue="tissue", palette=palette, ax=ax)
ax.set_title("Average distance between CD138 positive cells", fontsize=20)
plt.tight_layout()
plt.savefig(abspath("../out/mets-boxen-cd138distance.png"), dpi=300)
# Average distance of CD138 positive cells on Brain samples
metadata[metadata["tissue"]=="MBM"]["distances"].mean()
# Average distance of CD138 positive cells on Liver samples
metadata[metadata["tissue"]=="ECM"]["distances"].mean()<jupyter_output><empty_output><jupyter_text>Let's check the ratio of CD138 positive cells per image<jupyter_code>%%time
metadata["CD138_positive_cells"] = get_marker_positive_cells(folder_path, marker, cutoff)
metadata.head()
%%time
metadata["CD138_positive_cells_proportion"] = metadata["CD138_positive_cells"]/metadata["n"]
metadata.head()
%%time
plot_distribution_boxen(metadata, marker="CD138_positive_cells_proportion", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-CD138pos_cell_proportion.pdf"),
                             abspath("./out/mets-boxen-CD138pos_cell_proportion.png")])
plot_sample_distributions(metadata, x="sample", marker="CD138_positive_cells_proportion", hue="tissue", palette=palette,
                          out=[abspath("./out/mets-boxen-CD138pos_cell_proportion_by_sample.pdf"),
                               abspath("./out/mets-boxen-CD138pos_cell_proportion_by_sample.png")])
for g,d in metadata.groupby("tissue"):
    print(f"Average proportion of {marker}+ cells per image in {g} = {d['CD138_positive_cells_proportion'].mean():.4f}")<jupyter_output>Average proportion of CD138+ cells per image in ECM = 0.0045
Average proportion of CD138+ cells per image in MBM = 0.0168<jupyter_text>CD138 Immediate neighborhood 30, 40 and 50 micrometers<jupyter_code>%%time
# Add distances to metadata
metadata["CD138_positive_cells_in_neighborhood_30um"] = count_marker_positive_neighborhood(folder_path, marker, cutoff, distance=30)
metadata["CD138_positive_cells_in_neighborhood_40um"] = count_marker_positive_neighborhood(folder_path, marker, cutoff, distance=40)
metadata["CD138_positive_cells_in_neighborhood_50um"] = count_marker_positive_neighborhood(folder_path, marker, cutoff, distance=50)
print("Results for 30um")
for g, d in metadata.groupby("tissue"):
    print(f"Average # of cells in the immediate neighborhood of {marker}+ cells per image in {g} = {d['CD138_positive_cells_in_neighborhood_30um'].mean():.4f}")
print("Results for 40um")
for g, d in metadata.groupby("tissue"):
    print(f"Average # of cells in the immediate neighborhood of {marker}+ cells per image in {g} = {d['CD138_positive_cells_in_neighborhood_40um'].mean():.4f}")
print("Results for 50um")
for g, d in metadata.groupby("tissue"):
    print(f"Average # of cells in the immediate neighborhood of {marker}+ cells per image in {g} = {d['CD138_positive_cells_in_neighborhood_50um'].mean():.4f}")
# CD138 average neighborhood
plot_distribution_boxen(metadata, marker="CD138_positive_cells_in_neighborhood_30um", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-CD138_positive_cells_in_neighborhood_30um.pdf"),
                             abspath("./out/mets-boxen-CD138_positive_cells_in_neighborhood_30um.png")])

plot_distribution_boxen(metadata, marker="CD138_positive_cells_in_neighborhood_40um", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-CD138_positive_cells_in_neighborhood_40um.pdf"),
                             abspath("./out/mets-boxen-CD138_positive_cells_in_neighborhood_40um.png")])

plot_distribution_boxen(metadata, marker="CD138_positive_cells_in_neighborhood_50um", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-CD138_positive_cells_in_neighborhood_50um.pdf"),
                             abspath("./out/mets-boxen-CD138_positive_cells_in_neighborhood_50um.png")])<jupyter_output><empty_output><jupyter_text>CD138 neighborhood stats<jupyter_code># Wilcoxon ranksum test for CD138+ cells in neighborhood between MBM and ECM tissues
res_hood_30 = ranksums(metadata[metadata["tissue"]=="MBM"]["CD138_positive_cells_in_neighborhood_30um"], metadata[metadata["tissue"]=="ECM"]["CD138_positive_cells_in_neighborhood_30um"])
res_hood_30
res_hood_40 = ranksums(metadata[metadata["tissue"]=="MBM"]["CD138_positive_cells_in_neighborhood_40um"], metadata[metadata["tissue"]=="ECM"]["CD138_positive_cells_in_neighborhood_40um"])
res_hood_40
res_hood_50 = ranksums(metadata[metadata["tissue"]=="MBM"]["CD138_positive_cells_in_neighborhood_50um"], metadata[metadata["tissue"]=="ECM"]["CD138_positive_cells_in_neighborhood_50um"])
res_hood_50<jupyter_output><empty_output><jupyter_text>Check CD138 total expression per sample<jupyter_code>%%time
all_images = read_markers_labeled(folder_path, marker, metadata[["tissue", "sample"]])
all_images = all_images.sort_values(by="tissue", ascending=False)
all_images.head()
# Log scale the data
scaled = np.log(all_images[marker])
scaled = pd.DataFrame(scaled, columns=[marker])
scaled = scaled[~np.isinf(scaled[marker])]
scaled["tissue"] = all_images["tissue"]
scaled["sample"] = all_images["sample"]
scaled = scaled.sort_values(by="tissue", ascending=False)
scaled.head()
%%time
# Plot intensity distribution by tissue
plot_distributions(all_images, marker, hue="tissue", palette=palette,
                   out=[abspath("./out/mets-violin_CD138-color_MBM_ECM-raw.pdf"),
                        abspath("./out/mets-violin_CD138-color_MBM_ECM-raw.png")])
%%time
plot_distributions(scaled, marker, hue="tissue", palette=palette,
                            out=[abspath("./out/mets-violin_CD138-color_MBM_ECM-log.pdf"),
                                 abspath("./out/mets-violin_CD138-color_MBM_ECM-log.png")])
%%time
scaled_pos = scaled[scaled["CD138"]>=np.log(cutoff)]
scaled_pos
plot_distributions(scaled_pos, marker, hue="tissue", palette=palette,
                            out=[abspath("./out/mets-violin_CD138pos-intensity_by_tissue-log.pdf"),
                                 abspath("./out/mets-violin_CD138pos-intensity_by_tissue-log.png")])
%%time
# Plot violin plot for raw data
sns.set_style("darkgrid")
fig, ax = plt.subplots(1, 1, figsize=(20, 10))
sns.violinplot(data=scaled_pos, x="sample", y="CD138", hue="tissue", palette=palette, ax=ax)
ax.set_title("CD138+ Intensity", fontsize=20)
ax.tick_params(axis='x', labelrotation=90)
sns.despine(left=True)
plt.tight_layout()
fig.savefig(abspath("./out/mets-violin_CD138pos_intensity-by_sample_log.pdf"), dpi=300)
fig.savefig(abspath("./out/mets-violin_CD138pos_intensity-by_sample_log.png"), dpi=300)<jupyter_output>CPU times: total: 3.64 s
Wall time: 3.81 s<jupyter_text>Get CD138 mean marker intensities<jupyter_code>%%time
metadata["CD138_intensity"] = get_marker_intensities(folder_path, marker, cutoff=0)
metadata.head()
%%time
plot_distribution_boxen(metadata, "CD138_intensity", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-CD138_mean_intensity_by_tissue.pdf"),
                             abspath("./out/mets-boxen-CD138_mean_intensity_by_tissue.png")])
plot_sample_distributions(metadata, x="sample", marker="CD138_intensity", hue="tissue", palette=palette,
                          out=[abspath("./out/mets-scatter-CD138_mean_intensity_by_sample.pdf"),
                               abspath("./out/mets-scatter-CD138_mean_intensity_by_sample.png")])<jupyter_output><empty_output><jupyter_text>Get CD138+ mean marker intensities<jupyter_code>%%time
metadata["CD138pos_intensity"] = get_marker_intensities(folder_path, marker, cutoff=cutoff)
metadata.head()
plot_distribution_boxen(metadata, "CD138pos_intensity", hue="tissue", palette=palette,
                        out=[abspath("./out/mets-boxen-CD138pos_mean_intensity_by_tissue.pdf"),
                             abspath("./out/mets-boxen-CD138pos_mean_intensity_by_tissue.png")])
plot_sample_distributions(metadata, x="sample", marker="CD138pos_intensity", hue="tissue", palette=palette,
                          out=[abspath("./out/mets-scatter-CD138pos_mean_intensity_by_sample.pdf"),
                               abspath("./out/mets-scatter-CD138pos_mean_intensity_by_sample.png")])<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1351809954751131, 'fraction_numerical': 0.03959276018099547, 'mean_word_length': 5.418027828191168, 'pattern_counts': {'":': 2, '<': 46, '<?xml version=': 0, '>': 47, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '12', 'dir': 'jupyter-structured-clean-dedup', 'id': '558822', 'license': 'MIT', 'n_tokens_mistral': 4045, 'n_tokens_neox': 3581, 'n_words': 558, 'path': 'multiplexed_image_analysis/notebook/cd138.ipynb', 'repo_name': 'IzarLab/Melanoma_Brain_Metastasis', 'size': '10608'}