<jupyter_start><jupyter_text>function to load different data setsThis function also adds the Metdata columns Metadata_dataset, Metadata_partition and Metadata_features<jupyter_code>load_dataset  <- function(partition, dataset,feature){
    file_name  <- read_csv("../datasets.csv") 
    x  <-  file_name %>% filter(
         Partition == partition,
         Dataset == dataset,
         Features == feature) %>% 
         extract2("Link")

    return(read_csv(x) %>% 
          mutate(Metadata_dataset = dataset) %>%
          mutate(Metadata_partition = partition) %>% 
          mutate(Metadata_features = feature) 
          )
    }<jupyter_output><empty_output><jupyter_text>Load data We load training and test datasets for both BBBC036 / CDRP data set and select only important Metadata colums<jupyter_code># bbbc36 data 
bbbc036_train  <- load_dataset("Train","BBBC036","CellProfiler")  %>% 
    mutate(Metadata_x_mutation_status = "none")

bbbc036_test <- load_dataset("Test","BBBC036","CellProfiler")  %>% 
    mutate(Metadata_x_mutation_status = "none")

bbbc036  <- rbind(bbbc036_train, bbbc036_test)<jupyter_output><empty_output><jupyter_text>How large are the data sets?<jupyter_code>bbbc036_train %>% dim()
bbbc036_test %>% dim()<jupyter_output><empty_output><jupyter_text>What are the Metadata colums?<jupyter_code>bbbc036 %>% 
    select(Metadata_Plate, Metadata_Well, Metadata_Plate_Map_Name, Metadata_pert_id) %>% 
    slice(1:5) %>% 
    print()
bbbc036 %>% 
    select( Metadata_pert_id, Metadata_broad_sample_type,Metadata_dataset, Metadata_partition) %>% 
    slice(1:5) %>% 
    print()<jupyter_output># A tibble: 5 x 4
  Metadata_pert_id Metadata_broad_sample_ty… Metadata_dataset Metadata_partiti…
  <chr>            <chr>                     <chr>            <chr>            
1 BRD-K18250272    trt                       BBBC036          Train            
2 BRD-K18316707    trt                       BBBC036          Train            
3 BRD-K18438502    trt                       BBBC036          Train            
4 BRD-K18550767    trt                       BBBC036          Train            
5 BRD-K18574842    trt                       BBBC036          Train<jupyter_text>how many replicates do we have?<jupyter_code>bbbc036 %>% 
    group_by(Metadata_Plate_Map_Name, Metadata_pert_id) %>%
    summarise(n_groups = n()) %>%
    summarise(mean_replicates = mean(n_groups))
# bbbc022 data 
bbbc022_train  <- load_dataset("Train","BBBC022","CellProfiler")
 
bbbc022_test <- load_dataset("Test","BBBC022","CellProfiler")

bbbc022  <- rbind(bbbc022_train, bbbc022_test) %>%
  select(Metadata_Plate, Metadata_Well, Metadata_Plate_Map_Name, 
         Metadata_pert_id, Metadata_broad_sample_type,
         Metadata_dataset,Metadata_partition, 
         everything()
        )<jupyter_output>Parsed with column specification:
cols(
  Dataset = col_character(),
  Partition = col_character(),
  Features = col_character(),
  Link = col_character()
)
Parsed with column specification:
cols(
  .default = col_double(),
  Metadata_Plate = col_integer(),
  Metadata_Well = col_character(),
  Metadata_Assay_Plate_Barcode = col_integer(),
  Metadata_Plate_Map_Name = col_character(),
  Metadata_well_position = col_character(),
  Metadata_broad_sample = col_character(),
  Metadata_source_name = col_character(),
  Metadata_compound_name = col_character(),
  Metadata_smiles = col_character(),
  Metadata_solvent = col_character(),
  Metadata_pert_id = col_character(),
  Metadata_pert_mfc_id = col_character(),
  Metadata_pert_well = col_character(),
  Metadata_pert_id_vendor = col_character(),
  Metadata_cell_id = col_character(),
  Metadata_broad_sample_type = col_character(),
  Metadata_pert_vehicle = col_character(),
  Metadata_pert_type = col_character(),
  Metadata_exp = col_character()[...]<jupyter_text>How large are the training and test partitions for BBBC022?<jupyter_code>bbbc022_train %>% dim()
bbbc022_test %>% dim()
bbbc022 %>% 
    group_by(Metadata_Plate_Map_Name, Metadata_pert_id) %>%
    summarise(n_groups = n()) %>%
    summarise(mean_replicates = mean(n_groups))<jupyter_output><empty_output><jupyter_text>How large are the combined data sets?<jupyter_code>dim(bbbc022)
dim(bbbc036)<jupyter_output><empty_output><jupyter_text>Extract common features and common metadata<jupyter_code>colnames_bbbc022 <- colnames(bbbc022)
colnames_bbbc036 <- colnames(bbbc036)

Metadata_names_bbbc022 <- c(
   stringr::str_subset(colnames_bbbc022, "^Meta")
) 

Metadata_names_bbbc036 <- c(
   stringr::str_subset(colnames_bbbc036, "^Meta")
) 

common_metadata  <- intersect(Metadata_names_bbbc022, Metadata_names_bbbc036)  
common_features  <- setdiff(intersect(colnames_bbbc022, colnames_bbbc036),common_metadata)<jupyter_output><empty_output><jupyter_text>Concatenate data sets<jupyter_code>population  <- rbind(
    bbbc022 %>% 
        select(c(common_metadata, common_features)),
    bbbc036 %>% 
        select(c(common_metadata, common_features))
    ) %>% 
    mutate(Metadata_perturbation = 'chemical') %>% 
    select(Metadata_perturbation, everything())<jupyter_output><empty_output><jupyter_text>Important: update column names!<jupyter_code>colnames_combined  <- colnames(population)

common_metadata  <- c(
   stringr::str_subset(colnames_combined, "^Meta")
) 

common_features  <- setdiff(colnames_combined, common_metadata)<jupyter_output><empty_output><jupyter_text>Cytominer has problems handling column names '1', '2' so we rename them to 'Feature_1', ...<jupyter_code>common_features  <- paste0("Feature_",common_features)
colnames(population)  <- c(common_metadata, common_features)<jupyter_output><empty_output><jupyter_text>Normalize dataWe use cytominer to normalize both datasets with respect to the controls, i.e. EMPTY genes<jupyter_code>population_normalized  <- cytominer::normalize(
    population, 
    variables = common_features, 
    strata = c("Metadata_perturbation"), 
    sample = population %>% 
                filter(
                    Metadata_broad_sample == "DMSO",
                    Metadata_partition == "Train"
                ), 
    operation = "standardize"
)<jupyter_output><empty_output><jupyter_text>Aggregate data<jupyter_code>population_aggregated  <- cytominer::aggregate(
    population = population_normalized, 
    variables = common_features, 
    strata = c("Metadata_pert_id","Metadata_dataset","Metadata_partition"), 
    operation = "mean"
) 

population_aggregated %>% dim()
population_aggregated %>% filter(Metadata_dataset == "BBBC022") %>% dim()
population_aggregated %>% filter(Metadata_dataset == "BBBC036") %>% dim()<jupyter_output><empty_output><jupyter_text>Correlation matrix<jupyter_code>cor_matrix  <- cor(
    x = population_aggregated %>% 
        filter(Metadata_dataset == 'BBBC022') %>% 
        select(common_features) %>% 
        as.matrix() %>% 
        t, 
    y = population_aggregated %>% 
        filter(Metadata_dataset == 'BBBC036') %>% 
        select(common_features) %>% 
        as.matrix() %>% 
        t,
    use  = "complete.obs"
    ) 

cor_matrix %>% dim()<jupyter_output><empty_output><jupyter_text>Submision file<jupyter_code># set column names 
colnames(cor_matrix)  <- population_aggregated %>% 
                            filter(Metadata_dataset == 'BBBC036') %>%
                            extract2("Metadata_pert_id")

# set row names 
#rownames(cor_matrix)  <- population_aggregated %>% 
#                            filter(Metadata_dataset == 'BBBC036') %>%
#                            extract2("Metadata_broad_sample")#

cor_matrix %>% dim()
df  <- cor_matrix %>% as_data_frame() %>% 
            mutate(Metadata_pert_id = population_aggregated %>% 
                            filter(Metadata_dataset == 'BBBC022') %>%
                            extract2("Metadata_pert_id")) %>% 
            select(Metadata_pert_id, everything())

# write submission file
write.csv(df,"../cytodata-baseline_R_day_2_CP.csv",row.names = FALSE)
dim(df)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11210986267166043, 'fraction_numerical': 0.02759051186017478, 'mean_word_length': 3.8522107813446396, 'pattern_counts': {'":': 0, '<': 90, '<?xml version=': 0, '>': 116, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '15', 'dir': 'jupyter-structured-clean-dedup', 'id': '81084', 'license': 'MIT', 'n_tokens_mistral': 2849, 'n_tokens_neox': 2509, 'n_words': 414, 'path': 'cytodata-toolkit/R/Create-Submission_R-day2-CP.ipynb', 'repo_name': 'cytodata/cytodata-hackathon-2018', 'size': '8010'}