<jupyter_start><jupyter_text>Table of ContentsDependenciesFunctionsPathsMainLoad and cleanup estimate filesGDCDKFZCBTTCCreate an estimate_manifest filemerge estimate and manifest of KF and DKFZ by sample_iqmerge estimate and manifest of GDC by aliquot_iqSubset to primary tumours  Dependencies<jupyter_code>library(dplyr)
library(reshape2)
library(gtable)<jupyter_output>Attaching package: ‘dplyr’


The following objects are masked from ‘package:stats’:

    filter, lag


The following objects are masked from ‘package:base’:

    intersect, setdiff, setequal, union<jupyter_text>Functions to replace ids with barcodes using the map file (Provided by Komal)<jupyter_code>id2barcodefx <- function(inpath,estimate_outputname){
    estimate_output <- read.table(paste0(inpath, estimate_outputname), 
                              sep = "\t", header = TRUE, stringsAsFactors = FALSE)
    message("Currnet estimate file:")
    print(estimate_outputname)
    message("dimensions of estimate output file:")
    print(dim(estimate_output))
    estimate_output$SampleID <- as.character(estimate_output$SampleID)
# remove the X from sampleid
    estimate_output$SampleID <- gsub("X", "", estimate_output$SampleID)
# replace "." with "-"
    estimate_output$SampleID <- gsub("[.]", "-", estimate_output$SampleID)
    non_existing <- estimate_output[,1][!estimate_output[,1] %in% mappings$sample_id]
    non_existing <- as.character(non_existing)
    message("cannot find this many in mappings")
    print(length(non_existing))
    exp_matrix_subset <- estimate_output[!estimate_output[,1] %in% non_existing,]
    mappings$barcodes <- as.character(mappings$barcodes)
    exp_matrix_subset[,1] <- mappings$barcodes[match(exp_matrix_subset[,1],mappings$sample_id)]
    message("Dim after id2barcode replacement")
    print(dim(exp_matrix_subset))
    return(exp_matrix_subset)
}<jupyter_output><empty_output><jupyter_text>Paths<jupyter_code>manifestpath <- "/Users/anabbi/OneDrive - UHN/Documents/IPD2/Manifests/"
datapath <- "/Users/anabbi/OneDrive - UHN/Documents/IPD2/Data/"
plotpath <- "/Users/anabbi/OneDrive - UHN/Documents/IPD2/Plots/"<jupyter_output><empty_output><jupyter_text>Main<jupyter_code>load(paste0(manifestpath,"Master_manifest.RData"))
table(IPD_metadata$group)<jupyter_output><empty_output><jupyter_text>Load and cleanup estimate files  GDC Change TCGA and TARGET ids to barcodes<jupyter_code>mappings <- read.table(paste0(manifestpath,"sampleid_barcode_mappings.txt"),
                       sep = "\t", header = T, stringsAsFactors = FALSE)
mappings$barcodes <- as.character(mappings$sample_barcode)<jupyter_output><empty_output><jupyter_text>Got the output from pengbo<jupyter_code>NBL_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TARGET_NBL_ESTIMATEoutput.txt")
COAD_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TCGA_COAD_ESTIMATEoutput.txt")
LUAD_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TCGA_LUAD_ESTIMATEoutput.txt")
SKCM_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TCGA_SKCM_ESTIMATEoutput.txt")
PRAD_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TCGA_PRAD_ESTIMATEoutput.txt")
GBM_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TCGA_GBM_ESTIMATEoutput.txt")
LGG_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TCGA_LGG_ESTIMATEoutput.txt")
OV_estimate <- id2barcodefx(paste(datapath,"ESTIMATE/ESTIMATE_output/", sep = ""),
                             "TCGA_OV_ESTIMATEoutput.txt")<jupyter_output>Currnet estimate file:<jupyter_text>DKFZ<jupyter_code>ATRT_estimate <- read.table(paste0(datapath, "ESTIMATE/ESTIMATE_output/ICGC_ATRTs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt"),
                           sep = "\t", header = TRUE, stringsAsFactors = FALSE)

ET_estimate <- read.table(paste0(datapath, "ESTIMATE/ESTIMATE_output/ICGC_ETs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt"),
                           sep = "\t", header = TRUE, stringsAsFactors = FALSE)

MB_estimate <- read.table(paste0(datapath,"ESTIMATE/ESTIMATE_output/ICGC_MBs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt"),
                           sep = "\t", header = TRUE, stringsAsFactors = FALSE)

PA_estimate <- read.table(paste0(datapath,"ESTIMATE/ESTIMATE_output/ICGC_PAs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt"),
                           sep = "\t", header = TRUE, stringsAsFactors = FALSE)

pedGBM_estimate <- read.table(paste0(datapath, "ESTIMATE/ESTIMATE_output/ICGC_GBMs.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt"),
                           sep = "\t", header = TRUE, stringsAsFactors = FALSE)

PDX_estimate <- read.table(paste0(datapath, "ESTIMATE/ESTIMATE_output/PDX_2017Oct06.normalizedTPMs_perGencodeV26Gene.sleuth.estimateOutput.txt"),
                           sep = "\t", header = TRUE, stringsAsFactors = FALSE)
ATRT_estimate$SampleID <- gsub("[.]", "-", ATRT_estimate$SampleID)<jupyter_output><empty_output><jupyter_text>CBTTC<jupyter_code>CBTTC_estimate <- read.table(paste0(datapath, "ESTIMATE/ESTIMATE_output/ESTIMATEOutput.CBTTC.txt"),
                              sep = "\t", header = TRUE, check.names = F,
                            stringsAsFactors =F, row.names = NULL)
dim(CBTTC_estimate)
head(CBTTC_estimate)
colnames(CBTTC_estimate)[colnames(CBTTC_estimate) == "row.names"] <- "SampleID"
CBTTC_estimate$SampleID <- gsub("X", "",CBTTC_estimate$SampleID)
CBTTC_estimate$SampleID <- gsub("7316.", "7316-",CBTTC_estimate$SampleID)
head(CBTTC_estimate)<jupyter_output><empty_output><jupyter_text>Create an estimate_manifest file<jupyter_code>KF_DKFZ_estimate <- rbind(PA_estimate, #DKFZ
                        ET_estimate,
                        pedGBM_estimate,
                        ATRT_estimate,
                        MB_estimate,
                        CBTTC_estimate )<jupyter_output><empty_output><jupyter_text>merge estimate and manifest of KF and DKFZ by sample_iq<jupyter_code>colnames(KF_DKFZ_estimate)[colnames(KF_DKFZ_estimate) == "SampleID"] <- "sample_id"
KF_DKFZ_estimate_manifest <- merge(KF_DKFZ_estimate, IPD_metadata, by = "sample_id")
table(KF_DKFZ_estimate_manifest$group)<jupyter_output><empty_output><jupyter_text>merge estimate and manifest of GDC by aliquot_iq<jupyter_code>GDC_estimate <- rbind(NBL_estimate,
                     OV_estimate,
                     PRAD_estimate,
                     COAD_estimate,
                     SKCM_estimate,
                     GBM_estimate,
                     LGG_estimate,
                     LUAD_estimate)
colnames(GDC_estimate)[colnames(GDC_estimate) == "SampleID"] <- "aliquot_id"
GDC_estimate_manifest <- merge(GDC_estimate, IPD_metadata, by = "aliquot_id")<jupyter_output><empty_output><jupyter_text>17 TCGA samples missing from IPD_metadata, because their raw files were not available through cavatica, but their tpm file was forwarded to me by komal Merge both<jupyter_code>total_estimate_manifest <- rbind(GDC_estimate_manifest,
                                KF_DKFZ_estimate_manifest)<jupyter_output><empty_output><jupyter_text>Subset to primary tumours<jupyter_code>estimate_manifest_primary <- total_estimate_manifest[grepl("Initial",total_estimate_manifest$sample_type) |
                                                     grepl("Primary", total_estimate_manifest$sample_type), ]
estimate_manifest_primary <- rbind(estimate_manifest_primary,
                                  total_estimate_manifest[total_estimate_manifest$group == "KF_NBL",])
dim(estimate_manifest_primary)
table(total_estimate_manifest$group)
table(estimate_manifest_primary$group)
save(estimate_manifest_primary,
     file = paste0(datapath, "ESTIMATE/estimate_manifest_primary.RData"))   
save(PDX_estimate,
     file = paste0(datapath, "ESTIMATE/PDX_estimate.RData"))<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10755955274671852, 'fraction_numerical': 0.007170636849781235, 'mean_word_length': 3.99029714978775, 'pattern_counts': {'":': 0, '<': 94, '<?xml version=': 0, '>': 51, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 5, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '608496', 'license': 'Apache-2.0', 'n_tokens_mistral': 2925, 'n_tokens_neox': 2757, 'n_words': 374, 'path': 'notebooks/04_ESTIMATE_dataframe.ipynb', 'repo_name': 'pughlab/immunogenomics_pedNST', 'size': '8228'}