<jupyter_start><jupyter_text>Loading data<jupyter_code>import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import glob
import time
# mount google drive to access dataset food
from google.colab import drive
drive.mount('/content/drive', force_remount=True)
%cd /content/

!unzip '/content/drive/.shortcut-targets-by-id/1lp7vo6EG60jba1Gk_tuNpvDm7q5ZXlKX/Telecom_Specialization_ML3/Students/Peter_Christoph_Marko_Martin/data/archive.zip'
##### Inconsistency between file names and file names in meta data #####
## ==> rename files to the names used in meta_data_covid.csv
!mv "/content/curated_data/curated_data/2COVID/17_Zhao_Recurrence-of-positive-SARS-CoV-2-RNA-in-C_2020_International-Journal-of-Inf-p1-21%0.png" "/content/curated_data/curated_data/2COVID/17_Zhao_Recurrence-of-COVID-19-SARS-CoV-2-RNA-in-C_2020_International-Journal-of-Inf-p1-21%0.png"

!mv "/content/curated_data/curated_data/2COVID/17_Zhao_Recurrence-of-positive-SARS-CoV-2-RNA-in-C_2020_International-Journal-of-Inf-p1-21%1.png" "/content/curated_data/curated_data/2COVID/17_Zhao_Recurrence-of-COVID-19-SARS-CoV-2-RNA-in-C_2020_International-Journal-of-Inf-p1-21%1.png"
#17_Zhao_Recurrence-of-COVID-19-SARS-CoV-2-RNA-in-C_2020_International-Journal-of-Inf-p1-21%0
# Load data as dataframes
df_meta_cap = pd.read_csv('meta_data_cap.csv')
df_meta_covid = pd.read_csv('meta_data_covid.csv', encoding = "ISO-8859-1")
df_meta_normal = pd.read_csv('meta_data_normal.csv')<jupyter_output><empty_output><jupyter_text>Data analysis  Analysis of df_meta_cap<jupyter_code># Get some information about df_meta_cap
print(df_meta_cap.info())
display(df_meta_cap.head())

df_meta_cap.loc[df_meta_cap.duplicated(),:]  # get duplicates
df_meta_cap['Country'].unique()     # unique countries
df_meta_cap['Patient ID'].nunique() # number of unique patients
df_meta_cap['Age'].unique()         # unique ages
# Display age distribution
df_meta_cap.Age.plot(kind='hist')
plt.show()<jupyter_output><empty_output><jupyter_text>Analysis of df_meta_covid<jupyter_code>df_meta_covid.loc[df_meta_covid.duplicated(),:]  # get duplicates
# Get some information about df_meta_covid
print(df_meta_covid.info())
display(df_meta_covid.head())
df_meta_covid['Country'].unique()     # unique countries
df_meta_covid['Patient ID'].nunique() # number of unique patients<jupyter_output><empty_output><jupyter_text>Analysis of df_meta_normal<jupyter_code><jupyter_output><empty_output><jupyter_text>Split data into training, valdiation and test data<jupyter_code>np.random.seed(42)

# Configuration of sizes
test_size = 0.1
validation_size = 0.1

# List of classes
classes_list = [('1NonCOVID','NonCOVID'), 
                ('2COVID','COVID'), 
                ('3CAP','CAP')]
# Get patient to image mappings

# Covid patients
covid_patient_to_image = df_meta_covid.groupby(['Patient ID'])['File name'].apply(list).reset_index()
# Normal patients
normal_patient_to_image = df_meta_normal.groupby(['Patient ID'])['File name'].apply(list).reset_index()
# CAP patients
cap_patient_to_image = df_meta_cap.groupby(['Patient ID'])['File name'].apply(list).reset_index()
def split_ids(patient_to_image, val_size, test_size):
  '''
  '''
  # Calculate the number of training, valdiation and test samples
  len_all = len(patient_to_image)
  nb_test = np.floor(len_all * test_size).astype('int')
  nb_val = np.floor(len_all * val_size).astype('int')
  nb_train = len_all - (nb_test + nb_val).astype('int')
  # Create sub-samples of patient_to_image
  patient_ids = patient_to_image.index.values
  train_ids = np.random.choice(patient_ids, nb_train, replace=False)
  val_ids = np.random.choice(list(set(patient_ids)-set(train_ids)), nb_val, 
                             replace=False)
  test_ids = np.random.choice(list(set(patient_ids)-set(train_ids)-set(val_ids)),
                              nb_test, replace = False)
  
  return train_ids, val_ids, test_ids
# Split data into training, validation and test subsets

sample_id_dict = {}

for _, cl_dest in classes_list:
  if cl_dest == 'COVID':
    # Split covid patients
    covid_train_ids, covid_val_ids, covid_test_ids = \
      split_ids(covid_patient_to_image, validation_size, test_size)
    
    # Add data to dictionary
    sample_id_dict[cl_dest] = {'IMG':covid_patient_to_image,
                               'TRAIN':covid_train_ids,
                               'VAL':covid_val_ids,
                               'TEST':covid_test_ids}
  elif cl_dest == 'NonCOVID':
    # Split normal patients
    norm_train_ids, norm_val_ids, norm_test_ids = \
      split_ids(normal_patient_to_image, validation_size, test_size)
    # Add data to dictionary
    sample_id_dict[cl_dest] = {'IMG':normal_patient_to_image,
                               'TRAIN':norm_train_ids,
                               'VAL':norm_val_ids,
                               'TEST':norm_test_ids}
  else: # CAP
    # Split cap patients
    cap_train_ids, cap_val_ids, cap_test_ids = \
      split_ids(cap_patient_to_image, validation_size, test_size)
    # Add data to dictionary
    sample_id_dict[cl_dest] = {'IMG':cap_patient_to_image,
                               'TRAIN':cap_train_ids,
                               'VAL':cap_val_ids,
                               'TEST':cap_test_ids}


#print(len(covid_train_ids), len(covid_val_ids), len(covid_test_ids))
#print(len(norm_train_ids), len(norm_val_ids), len(norm_test_ids))
#print(len(cap_train_ids), len(cap_val_ids), len(cap_test_ids))
# Create directories
source_dir = '/content/curated_data/curated_data/'
train_dir = str(source_dir + 'train/')
test_dir = str(source_dir + 'test/')
validation_dir = str(source_dir + 'validation/')
!mkdir $train_dir
!mkdir $test_dir
!mkdir $validation_dir

# Create subdirectories for each class
for _, cl in classes_list:
  c_train = str(train_dir + cl)
  !mkdir $c_train
  c_test = str(test_dir + cl)
  !mkdir $c_test
  c_validation = str(validation_dir + cl)
  !mkdir $c_validation
def move_files(patient_to_image, patient_ids, from_dir, to_dir):
  '''
  '''
  # Set a limit to mv-strings
  MAX_LEN = 20000

  # Check if at least one patient id was provided
  if len(patient_to_image) < 1:
    print('No files to move!')
    return None

  fs_to_move_stack = []
  fs_to_move = ''
  for id in patient_ids:
    file_names = f' {from_dir}'.join([fn for fn in patient_to_image.at[id,'File name']])
    # Extend fs_to_move by new file names and from_dir at the beginning
    fs_to_move = fs_to_move + f' {from_dir}' + file_names
    if len(fs_to_move) > MAX_LEN:
      fs_to_move_stack.append(fs_to_move)
      fs_to_move = ''
  # Append the last fs_to_move string to the file_stack
  fs_to_move_stack.append(fs_to_move)
  for fs_to_move in fs_to_move_stack:
    !mv $fs_to_move $to_dir 

# Move all data to new destinations
%%time

for cl_src, cl_dest in classes_list:
  from_dir = source_dir + cl_src + '/'
  
  imgs = sample_id_dict[cl_dest]['IMG']

  # Move training data
  to_dir = train_dir + cl_dest
  ids = sample_id_dict[cl_dest]['TRAIN']
  move_files(imgs, ids, from_dir, to_dir)
  #print(to_dir)
  #print(len(ids))

  # Move validation data
  to_dir = validation_dir + cl_dest
  ids = sample_id_dict[cl_dest]['VAL']
  move_files(imgs, ids, from_dir, to_dir)
  #print(to_dir)
  #print(len(ids))

  # Move test data
  to_dir = test_dir + cl_dest
  ids = sample_id_dict[cl_dest]['TEST']
  move_files(imgs, ids, from_dir, to_dir)
  #print(to_dir)
  #print(len(ids))<jupyter_output>CPU times: user 334 ms, sys: 480 ms, total: 814 ms
Wall time: 6.95 s<jupyter_text>lightGBM<jupyter_code>from timeit import default_timer as timer
import os
import cv2
import numpy as np
import glob
source_dir = '/content/curated_data/curated_data/'
train_dir = str(source_dir + 'train/')
test_dir = str(source_dir + 'test/')
validation_dir = str(source_dir + 'validation/')
data_format = '/*.png'
!find '/content/curated_data/curated_data/test' -type f | wc -l
!find '/content/curated_data/curated_data/train' -type f | wc -l
!find '/content/curated_data/curated_data/validation' -type f | wc -l
#dict_case_df.keys()
dict_case_lst = ['NonCOVID', 'COVID', 'CAP']<jupyter_output><empty_output><jupyter_text>train<jupyter_code>def load_imagelist_label(dir):
  lgbm_list = []
  lgbm_label = []

  for case in dict_case_lst:#dict_case_df.keys():
    file_list = glob.glob(os.path.join(dir, case + data_format))
    
    for f in range(len(file_list)):  
      img = cv2.imread(file_list[f], cv2.IMREAD_GRAYSCALE)
      img = cv2.resize(img, (224, 224))
      lgbm_list.append(img)
      lgbm_label.append(case)

  return lgbm_list,lgbm_label    

lgbm_list,lgbm_label = load_imagelist_label(train_dir)  
lgbm_array = np.array(lgbm_list)
lgbm_array.shape
lgbm_array.shape
lgbm_array = lgbm_array / 255.0
lgbm_array.shape
lgbm_label = np.array(lgbm_label)
#from sklearn.linear_model import LogisticRegression

#clf = LogisticRegression(multi_class='ovr')
lgbm_array.reshape(lgbm_array.shape[0], -1).shape
#clf.fit(lr_array.reshape(lr_array.shape[0], -1), lr_label)<jupyter_output><empty_output><jupyter_text>fit<jupyter_code>import lightgbm as lgb

#lgbm = lgb.LGBMClassifier(random_state=42)
from timeit import default_timer as timer
start = timer()
#from sklearn.model_selection import GridSearchCV

#lgbm = lgb.LGBMClassifier(**params,random_state=42)

#params = {"learning_rate": [0.01],#[0.01, 0.05, 0.1]
#          "num_leaves": [10],#[10, 20, 30]
#          "max_depth": [5],#[2, 5, 10]
#          "n_estimators": [50]}#[50, 100, 150]

#grid_search = GridSearchCV(lgbm, n_jobs=-1, param_grid=params, cv=3, scoring="neg_log_loss")
#grid_search.fit(lgbm_array.reshape(lgbm_array.shape[0], -1), lgbm_label)
#callbacks=[lgb.reset_parameter(learning_rate= lambda current_round: 0.1 * np.exp(-0.001 * current_round))]
#grid_search.best_params_
#grid_search.best_score_
#lgbm = grid_search.best_estimator_
#compute_logloss(lgbm, X_train, X_test, y_train, y_test)
params = {#'learning_rate': 0.05, 
          'max_depth': 5, 
          'num_leaves': 10, 
          'n_estimators': 97}
          #,'objective': 'binary'}
#lgbm.fit(lgbm_array.reshape(lgbm_array.shape[0], -1), lgbm_label,callbacks=[lgb.reset_parameter(learning_rate= lambda current_round: 0.1 * np.exp(-0.001 * current_round))])
lgbm = lgb.LGBMClassifier(**params,random_state=42)
lgbm.fit(lgbm_array.reshape(lgbm_array.shape[0], -1), lgbm_label)<jupyter_output><empty_output><jupyter_text>LGBMClassifier(boosting_type='gbdt', class_weight=None, colsample_bytree=1.0,               importance_type='split', learning_rate=0.1, max_depth=-1,               min_child_samples=20, min_child_weight=0.001, min_split_gain=0.0,               n_estimators=100, n_jobs=-1, num_leaves=31, objective=None,               random_state=42, reg_alpha=0.0, reg_lambda=0.0, silent=True,               subsample=1.0, subsample_for_bin=200000, subsample_freq=0)<jupyter_code>end = timer()

print(f"Training LGBM took {end-start: .2f} seconds")
#shoud stop here,
#try train validat than test overriding variable memory removing 
 y_pred = lgbm.predict(lgbm_array.reshape(lgbm_array.shape[0], -1))
y_pred
len(lgbm_label)
import sklearn.metrics as met
met.accuracy_score(lgbm_label, y_pred)
import sklearn.metrics as met
#def compute_logloss_train(model, X_train,y_train):
  
  #print(f"Log loss train set: {met.log_loss(y_train, model.predict_proba(X_train)[:,1]): .4f}")
  #print(f"Log loss test set: {met.log_loss(y_test, model.predict_proba(X_test)[:,1]): .4f}")

  ### martin add start
  #print("\n add by martin")
  #print(f"Training accuracy: \t{met.accuracy_score(y_train, model.predict(X_train)) : .4f}")
  #print(f"Test accuracy: \t\t{met.accuracy_score(y_test,  model.predict(X_test)): .4f}")
#compute_logloss_train(lgbm,lgbm_array.reshape(lgbm_array.shape[0], -1),lgbm_label)
y_train =  lgbm_label<jupyter_output><empty_output><jupyter_text>test<jupyter_code>test_dir
lgbm_list,lgbm_label = load_imagelist_label(test_dir) 
lgbm_array_test = np.array(lgbm_list)
lgbm_array_test.shape
lgbm_array_test = lgbm_array_test / 255.0
lgbm_array_test.shape
lgbm_label_test = np.array(lgbm_label)
lgbm_array_test.reshape(lgbm_array_test.shape[0], -1).shape
 y_pred_test = lgbm.predict(lgbm_array_test.reshape(lgbm_array_test.shape[0], -1))
met.accuracy_score(lgbm_label_test, y_pred_test)<jupyter_output><empty_output><jupyter_text>validate<jupyter_code>validation_dir
lgbm_list,lgbm_label = load_imagelist_label(validation_dir) 
lgbm_array_validation = np.array(lgbm_list)
lgbm_array_validation = lgbm_array_validation / 255.0
lgbm_label_validation = np.array(lgbm_label)
lgbm_array_validation.reshape(lgbm_array_validation.shape[0], -1).shape
 y_pred_validation = lgbm.predict(lgbm_array_validation.reshape(lgbm_array_validation.shape[0], -1))
met.accuracy_score(lgbm_label_validation, y_pred_validation)
import sklearn.metrics as met
def compute_logloss_train(model, X_train, X_test, y_train, y_test):
  
  print(f"Log loss train set: {met.log_loss(y_train, model.predict_proba(X_train)[:,1]): .4f}")
  print(f"Log loss test set: {met.log_loss(y_test, model.predict_proba(X_test)[:,1]): .4f}")

  ### martin add start
  print("\n add by martin")
  print(f"Training accuracy: \t{met.accuracy_score(y_train, model.predict(X_train)) : .4f}")
  print(f"Test accuracy: \t\t{met.accuracy_score(y_test,  model.predict(X_test)): .4f}")
  ### martin add stop
#compute_logloss(lgbm,lgbm_array.reshape(lr_array.shape[0], -1),lgbm_array_test.reshape(lgbm_array_test.shape[0], -1),lgbm_label,lgbm_label_test)
y_pred_train = lgbm.predict(lgbm_array.reshape(lgbm_array.shape[0], -1))
y_pred_test = lgbm.predict(lgbm_array_test.reshape(lgbm_array_test.shape[0], -1))
y_prob_train = lgbm.predict_proba(lgbm_array.reshape(lgbm_array.shape[0], -1))[:,1]
y_prob_test = lgbm.predict_proba(lgbm_array_test.reshape(lgbm_array_test.shape[0], -1))[:,1]

#print(f"Log loss training set: {met.log_loss(df_train['target'], y_prob_train)}")
#print(f"Log loss test set: {met.log_loss(df_test['target'], y_prob_test)}")
from tensorflow.keras import datasets, layers, models
import matplotlib.pyplot as plt
import tensorflow as tf
import numpy as np
import seaborn as sns
import pandas as pd

def plot_confusion_matrix_x(labels,predictions,classes_name):
  """
  example:
  predictions=max_predict # np.argmax(y_predict, axis=-1)
  labels=test_generator.classes
  classes_name = artist_list
  """
  con_mat = tf.math.confusion_matrix(labels=labels, predictions=predictions).numpy()
  con_mat_norm = np.around(con_mat.astype('float') / con_mat.sum(axis=1)[:, np.newaxis], decimals=2)

  con_mat_df = pd.DataFrame(con_mat_norm,index = classes_name,columns = classes_name)

  figure = plt.figure(figsize=(8, 8))
  sns.heatmap(con_mat_df, annot=True,cmap=plt.cm.Blues)
  plt.tight_layout()
  plt.ylabel('True label')
  plt.xlabel('Predicted label')
  plt.show()
lgbm_label_test
dict_case_lst
y_pred_test

from sklearn.metrics import classification_report
report = classification_report(lgbm_label_test, y_pred_test, target_names=dict_case_lst)
print(report)
from sklearn.metrics import plot_confusion_matrix
plot_confusion_matrix(clf, lr_array.reshape(lr_array.shape[0], -1), lr_label, cmap=plt.cm.Blues, normalize='true')  
plt.show()  
plot_confusion_matrix(lgbm_label_test, y_pred_test, labels=dict_case_lst, cmap=plt.cm.Blues, normalize='true')  
plt.show() 
plot_confusion_matrix_x(lgbm_label_test,y_pred_test,dict_case_lst)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1260602136613791, 'fraction_numerical': 0.018128844286176757, 'mean_word_length': 5.166067864271457, 'pattern_counts': {'":': 4, '<': 45, '<?xml version=': 0, '>': 46, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '11', 'dir': 'jupyter-structured-clean-dedup', 'id': '111722', 'license': 'MIT', 'n_tokens_mistral': 6072, 'n_tokens_neox': 5850, 'n_words': 969, 'path': 'covid_19_V1_2_P_New_Base_Martin_cleaning.ipynb', 'repo_name': 'martinfinis/HelloCodeSchoolProject', 'size': '15445'}