<jupyter_start><jupyter_text>Reduce the sample size.In this notebook, we reduce the sample size to make the analysis more approachable using average infrastructure.<jupyter_code>from __future__ import print_function
import pandas as pd
import seaborn as sns
from matplotlib import pyplot
import matplotlib

import warnings
warnings.filterwarnings('ignore')
sns.set_style("darkgrid")
sns.set(font_scale=1.5)

sns.set_context('paper')

# %matplotlib notebook

import pandas as pd

data_location = "../../data/raw/"<jupyter_output><empty_output><jupyter_text>TSV derived from the VCF dataset<jupyter_code># For a cohort of 2749 samples (TB-portals + HTBC)

# ROWS - inclusion of INDELS lead to 96633 more data points
# cat cohort.bqsr.filter.snps.tsv  | wc -l => 356154
# cat cohort.bqsr.filter.snps.indels.tsv  | wc -l => 452787

# In the reduced dataset 1726 samples
# cat final.cohort.bqsr.filter.snps.tsv | wc -l => 142327

# In the final filtered dataset for 1726 samples
# head -1 final.binarized_final_monolabel_df.tsv | sed 's/[^\t]//g'| wc -c => 52684
# libraries
import numpy as np
import matplotlib.pyplot as plt
 

plt.figure(
    figsize=(10,10),
    dpi=100)

# set width of bar
barWidth = 0.25
 
# set height of bar
bars1 = [800, 423]
bars2 = [338, 163]


# Set position of bar on X axis
r1 = np.arange(len(bars1))
r2 = [x + barWidth for x in r1]
r3 = [x + barWidth for x in r2]
 
# Make the plot
plt.bar(r1, bars1, width=barWidth, edgecolor='white', label='TBP')
plt.bar(r2, bars2, width=barWidth, edgecolor='white', label='HTBC')
 
# Add xticks on the middle of the group bars
# plt.xlabel('Cohort Drug Resistance Profile', fontweight='bold')
plt.xticks([r + 0.10 for r in range(len(bars1))], ['Drug Resistant Samples', 'Drug Sensitive Samples'])
# plt.xticks(rotation=45)

pyplot.ylabel('Number of samples') 


# Create legend & Show graphic
plt.legend()
plt.show()


a4_dims = (15, 10)
fig, ax = pyplot.subplots(figsize=a4_dims) 
sns.barplot(ax=ax, x=['cohort.bqsr.filter.snps.indels.tsv', 'cohort.bqsr.filter.snps.tsv', 'final.cohort.bqsr.filter.snps.tsv', 'final.binarized_final_monolabel_df.tsv' ], y=[452787, 356154, 142327, 52684 ])
 

# libraries
import numpy as np
import matplotlib.pyplot as plt
plt.style.use('seaborn-darkgrid')


plt.figure(
    figsize=(10,10),
    dpi=100)

# set width of bar
barWidth = 0.25
 
# set height of bar
bars1 = [ 142327, 52684 ]
bars2 = [338, 163]


# Set position of bar on X axis
r1 = np.arange(len(bars1))
 
# Make the plot
plt.bar(r1, bars1, width=barWidth, edgecolor='white')
# plt.bar(r2, bars2, width=barWidth, edgecolor='white', label='HTBC')
 
# Add xticks on the middle of the group bars
# plt.xlabel('Cohort Drug Resistance Profile', fontweight='bold')
plt.xticks([0.05, 1], ['Raw Mutation Dataset', 'Processed Mutation Dataset'], fontsize=14)
# plt.xticks(rotation=45)

plt.yticks(fontsize=14)
plt.ylabel('Number of columns', fontsize=14) 




# Create legend & Show graphic
# plt.legend()
plt.show()


a4_dims = (10, 10)
fig, ax = pyplot.subplots(figsize=a4_dims) 
sns.barplot(ax=ax, x=['final.cohort.bqsr.filter.snps.tsv', 'final.binarized_final_monolabel_df.tsv' ], y=[ 142327, 52684 ])<jupyter_output><empty_output><jupyter_text>Tb-profiler dataset<jupyter_code># Read and transpose the cohort tbprofile file for ALL TB-portals + HTBC Genomes with SampleID as index


tbprofiler_df = pd.read_json(data_location + "cohort.tbprofiler.json", encoding="UTF-8")
tbprofiler_df = tbprofiler_df.transpose()
tbprofiler_df.head()

tbprofiler_df.shape

tbprofiler_df.columns.tolist()
# Derive a new column `Resistance_Status` by treating MDR/XDR/TDR as the same and preparing Tb-profiler output for Binary Classification

resistance_status_df = tbprofiler_df
resistance_status_df['Resistance_Status'] = resistance_status_df.apply(
    lambda row: 'Sensitive' if (row.drtype == 'Sensitive') else 'Resistant', axis=1)
# resistance_status_df.head()


drugs_column_names = ['rifampicin',
                      'isoniazid',
                      'pyrazinamide',
                      'ethambutol',
                      'streptomycin',
                      'fluoroquinolones',
                      'moxifloxacin',
                      'ofloxacin',
                      'levofloxacin',
                      'ciprofloxacin',
                      'aminoglycosides',
                      'amikacin',
                      'kanamycin',
                      'capreomycin',
                      'ethionamide',
                      'para-aminosalicylic_acid',
                      'cycloserine',
                      'linezolid',
                      'bedaquiline',
                      'clofazimine',
                      'delamanid']

lineage_column_names = ['main_lin', 'sublin']

resistance_status_column_names = ['drtype', 'MDR', 'XDR', 'Resistance_Status']

renamed_drug_columns_names = ['rifampicin_resistance',
                              'isoniazid_resistance',
                              'pyrazinamide_resistance',
                              'ethambutol_resistance',
                              'streptomycin_resistance',
                              'fluoroquinolones_resistance',
                              'moxifloxacin_resistance',
                              'ofloxacin_resistance',
                              'levofloxacin_resistance',
                              'ciprofloxacin_resistance',
                              'aminoglycosides_resistance',
                              'amikacin_resistance',
                              'kanamycin_resistance',
                              'capreomycin_resistance',
                              'ethionamide_resistance',
                              'para-aminosalicylic_acid_resistance',
                              'cycloserine_resistance',
                              'linezolid_resistance',
                              'bedaquiline_resistance',
                              'clofazimine_resistance',
                              'delamanid_resistance']

renamed_drug_columns_names_dict = {
    'rifampicin': 'rifampicin_resistance',
    'isoniazid': 'isoniazid_resistance',
    'pyrazinamide': 'pyrazinamide_resistance',
    'ethambutol': 'ethambutol_resistance',
    'streptomycin': 'streptomycin_resistance',
    'fluoroquinolones': 'fluoroquinolones_resistance',
    'moxifloxacin': 'moxifloxacin_resistance',
    'ofloxacin': 'ofloxacin_resistance',
    'levofloxacin': 'levofloxacin_resistance',
    'ciprofloxacin': 'ciprofloxacin_resistance',
    'aminoglycosides': 'aminoglycosides_resistance',
    'amikacin': 'amikacin_resistance',
    'kanamycin': 'kanamycin_resistance',
    'capreomycin': 'capreomycin_resistance',
    'ethionamide': 'ethionamide_resistance',
    'para-aminosalicylic_acid': 'para-aminosalicylic_acid_resistance',
    'cycloserine': 'cycloserine_resistance',
    'linezolid': 'linezolid_resistance',
    'bedaquiline': 'bedaquiline_resistance',
    'clofazimine': 'clofazimine_resistance',
    'delamanid': 'delamanid_resistance'
}

# Rename the columns for being explicit once the overall dataframe is created.

resistance_status_df.rename(columns=renamed_drug_columns_names_dict,
                            inplace=True)

resistance_status_df.head()
# Binarize the dataframe, for drug columns, to focus only on the presence/absence of mutation.
# We treat all "-" as absence of mutation and assign it as 0

binarized_resistance_status_df = resistance_status_df

for col_name in renamed_drug_columns_names:
    binarized_resistance_status_df[col_name] = resistance_status_df[col_name].apply(
        lambda resistance: 0 if resistance is '-' else 1)

binarized_resistance_status_df.head()
binarized_resistance_status_df.shape

binarized_resistance_status_df[['Resistance_Status']].head()
binarized_resistance_status_df[['Resistance_Status']]
binarized_resistance_status_df[renamed_drug_columns_names].sum().to_dict()
a_dict = binarized_resistance_status_df[renamed_drug_columns_names].sum().to_dict()

# sns.barplot(data=a_dict, y=a_dict.values())
a_dict
# Distribution of Resistant/Sensitive genomes

binarized_resistance_status_df['Resistance_Status'].hist()

# Isolate a list of resistant genomes

resistant_genomes = binarized_resistance_status_df[binarized_resistance_status_df['Resistance_Status'] == "Resistant"].index.tolist()
resistant_genomes
# Isolate a list of sensitive genomes

sensitive_genomes = binarized_resistance_status_df[binarized_resistance_status_df['Resistance_Status'] == "Sensitive"].index.tolist()
sensitive_genomes
# Read the genomes IDs of genomes
# - Tb-portals
# - HTBC

import json

with open(data_location + "test_train_genome_ids.json") as f:
    all_genomes_ids_dict = json.load(f)

all_genomes_ids_dict

# Genome IDs from the tb-portals dataset
tbportals_genomes = all_genomes_ids_dict['tb_portals']

len(tbportals_genomes)
tbportals_genomes_set = set(tbportals_genomes)

tbportals_genomes_set
# Genome IDs from the htbc dataset
htbc_genomes = set(all_genomes_ids_dict['htbc'])

len(htbc_genomes)

htbc_genomes_set = set(htbc_genomes)
# Find HTBC genomes, which are resistant

htbc_resistant_genomes = list(htbc_genomes_set.intersection(resistant_genomes))
len(htbc_resistant_genomes)

# Find HTBC genomes, which are sensitive

htbc_sensitive_genomes = list(htbc_genomes_set.intersection(sensitive_genomes))
len(htbc_sensitive_genomes)
# Find Tb-portals genomes, which are resistant
tbportals_resistant_genomes  = list(tbportals_genomes_set.intersection(resistant_genomes))
len(tbportals_resistant_genomes[830:])
# Find Tb-portals genomes, which are sensitive
tbportals_sensitive_genomes = list(tbportals_genomes_set.intersection(sensitive_genomes))
len(tbportals_sensitive_genomes)
# Create a list of HTBC genomes to be used for the final analysis.

final_htbc_test_genome_ids = htbc_resistant_genomes + htbc_sensitive_genomes
len(final_htbc_test_genome_ids)
# Create a list of Tb-portals genomes to be used for the final analysis.

final_tbportals_train_genome_ids = tbportals_resistant_genomes[830:] + tbportals_sensitive_genomes[100:]
len(final_tbportals_train_genome_ids)

# Append the list of final Tb-portals genomes to the JSON file.

all_genomes_ids_dict['final_tbportals_train_genomes'] = final_tbportals_train_genome_ids
all_genomes_ids_dict.keys()


# Append the list of final HTBC genomes to the JSON file.

all_genomes_ids_dict['final_htbc_test_genomes'] = final_htbc_test_genome_ids
all_genomes_ids_dict.keys()

# Write the final JSON file.
import json

with open(data_location + "test_train_genome_ids.json", "w") as f:
    json.dump(all_genomes_ids_dict, f, indent= 4)<jupyter_output><empty_output><jupyter_text>Create main monolabel dataset and segregate it into- train dataset- test dataset  Read the output of step-001 and derive the monolabel dataset<jupyter_code>binarized_final_df = pd.read_csv("../data/processed/final.binarized_final_multilabel_df.csv").set_index('SampleID')


binarized_final_df.head()
binarized_final_df.columns
binarized_final_df.index
binarized_final_df= binarized_final_df.drop(columns=[*renamed_drug_columns_names, *lineage_column_names, 'drtype', 'MDR', 'XDR'], axis= 1)

binarized_final_df.head()
binarized_final_df.shape
binarized_final_df['Resistance_Status']= binarized_final_df['Resistance_Status'].apply(lambda resistance: 0.0 if resistance == 'Sensitive' else 1.0)
binarized_final_df.head()
binarized_final_df.to_csv("../data/processed/final.binarized_final_monolabel_df.tsv", "\t")
binarized_final_df.head()


train = binarized_final_df.loc[final_tbportals_train_genome_ids]
train.shape

train.to_csv("../data/processed/final.train.tsv", "\t")
train.head()


test = binarized_final_df.loc[final_htbc_test_genome_ids]
test.shape

test.to_csv("../data/processed/final.test.tsv", "\t")
test.shape<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.10115973927029544, 'fraction_numerical': 0.016845847794802336, 'mean_word_length': 3.8477636438243743, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 21, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '197659', 'license': 'BSD-3-Clause', 'n_tokens_mistral': 4298, 'n_tokens_neox': 3881, 'n_words': 785, 'path': 'notebooks/FINAL/002_choose_limited_tbportals_genomes.ipynb', 'repo_name': 'abhi18av/LJMU_Masters_Dissertation', 'size': '11813'}