<jupyter_start><jupyter_text>[View in Colaboratory](https://colab.research.google.com/github/myfellowship/myfellowship/blob/master/Predicting%20nodule%20on%20X%20ray.ipynb)<jupyter_code>!pip install kaggle
from googleapiclient.discovery import build
import io, os
from googleapiclient.http import MediaIoBaseDownload
from google.colab import auth

auth.authenticate_user()

drive_service = build('drive', 'v3')
results = drive_service.files().list(
        q="name = 'kaggle.json'", fields="files(id)").execute()
kaggle_api_key = results.get('files', [])

filename = "/content/.kaggle/kaggle.json"
os.makedirs(os.path.dirname(filename), exist_ok=True)

request = drive_service.files().get_media(fileId=kaggle_api_key[0]['id'])
fh = io.FileIO(filename, 'wb')
downloader = MediaIoBaseDownload(fh, request)
done = False
while done is False:
    status, done = downloader.next_chunk()
    print("Download %d%%." % int(status.progress() * 100))
os.chmod(filename, 600)
!kaggle datasets download -d nih-chest-xrays/data -f images_001.zip
!ls
import zipfile

def unzipping(file):
  with zipfile.ZipFile(file,"r") as zip_ref:
    zip_ref.extractall("data")
  os.remove(file)
unzipping("images_001.zip")
!ls
files = os.listdir('data/images')
len(files)
!kaggle datasets download -d nih-chest-xrays/data -f images_002.zip
unzipping("images_002.zip")
files = os.listdir('data/images')
len(files)<jupyter_output><empty_output><jupyter_text>getting the list of positive files<jupyter_code>import pandas as pd
data = pd.read_csv('https://raw.githubusercontent.com/gvsanthu10/Dicom/master/onlypositive.csv')
data.head()
file_list= data['ID'].values
file_list
len(file_list)
os.getcwd()
os.chdir('data')
for the_file in os.listdir('images'):
  if the_file in file_list:
    pass
  else:
    os.remove(os.path.join('images', the_file))
files = os.listdir('images')
len(files)<jupyter_output><empty_output><jupyter_text>**I reduced the number of files from 14999 to 9833**<jupyter_code>files= sorted(files)
file_list<jupyter_output><empty_output><jupyter_text>**both list are maching**<jupyter_code>import matplotlib.pyplot as plt
import matplotlib.image as mpimg
import numpy as np
os.chdir('images')
img=mpimg.imread('00000002_000.png')
imgplot = plt.imshow(img)
plt.show()<jupyter_output><empty_output><jupyter_text>**this is was normal as from the list**, matching **Now starting binary classification**<jupyter_code>os.getcwd()
os.chdir('..')
os.chdir('..')<jupyter_output><empty_output><jupyter_text>*Making labels*<jupyter_code>labels = np.array(data['nodule'])
labels<jupyter_output><empty_output><jupyter_text>Making data<jupyter_code>os.getcwd()
path = 'data/images/'
from PIL import Image
import cv2 

data=[]

for i in files:
    imag=cv2.imread(os.path.join(path, i))
    img_from_ar = Image.fromarray(imag)
    resized_image = img_from_ar.resize((200, 200))
    data.append(np.array(resized_image))
    
xrays=np.array(data)

xrays.shape
labels=labels[:3117]
labels.shape
np.save("xrays",xrays)
np.save("labels",labels)

#we can load these back later
#np.load("xrays",xrays)
#np.load("labels",labels)
#suffuling

s=np.arange(xrays.shape[0])
np.random.shuffle(s)
xrays=xrays[s]
labels=labels[s]
#nuber of categories and number of data (whcih is 2 and 3117)

num_classes=len(np.unique(labels))
data_length=len(xrays)
#diving xrays to train test

(x_train,x_test)=xrays[(int)(0.1*data_length):],xrays[:(int)(0.1*data_length)]

x_train = x_train.astype('float32')/255
x_test = x_test.astype('float32')/255

train_length=len(x_train)
test_length=len(x_test)
#diving labels to train test

(y_train,y_test)=labels[(int)(0.1*data_length):],labels[:(int)(0.1*data_length)]
#one hot encoder

import keras
from keras.utils import np_utils

#One hot encoding
y_train=keras.utils.to_categorical(y_train,num_classes)
y_test=keras.utils.to_categorical(y_test,num_classes)<jupyter_output><empty_output><jupyter_text>Now making  a model<jupyter_code>x_train.shape
dataDim = np.prod(xrays[0].shape)
def flatten(dimData, xrays):
    xrays = np.array(xrays)
    xrays = xrays.reshape(len(xrays), dimData)
    return xrays
x_train  = flatten(dataDim, x_train)
x_test = flatten(dataDim, x_test)
from keras.models import Sequential
from keras.layers import Dense
from keras.layers import Dropout

#three layers
#activation function: both
#neurons: 256
model = Sequential()
model.add(Dense(256, activation = 'tanh', input_shape = (dataDim,)))
model.add(Dropout(0.2))
model.add(Dense(256, activation='tanh'))
model.add(Dropout(0.2))
model.add(Dense(256, activation='relu'))
model.add(Dropout(0.2))
model.add(Dense(2, activation='softmax'))
model.compile(optimizer='rmsprop', loss='categorical_crossentropy', metrics=['accuracy'])
history = model.fit(x_train, y_train, batch_size = 50, epochs=10, verbose=1,
                    validation_data=(x_train, y_train))
[test_loss, test_acc] = model.evaluate(x_test, y_test)
print("Evaluation result on Test Data : Loss = {}, accuracy = {}".format(test_loss, test_acc))<jupyter_output>311/311 [==============================] - 0s 667us/step
Evaluation result on Test Data : Loss = 0.38766746956052506, accuracy = 0.8745980715061691<jupyter_text>Predicting<jupyter_code>from PIL import Image
import cv2 

data=[]

for i in range(4):
    imag=cv2.imread(os.path.join(path, files[i]))
    img_from_ar = Image.fromarray(imag)
    resized_image = img_from_ar.resize((200, 200))
    data.append(np.array(resized_image))
xrays=np.array(data)
xrays.shape
xrays = xrays.astype('float32')/255
for i in range(5):
  img=mpimg.imread(os.path.join('data/images/')+files[i])
  imgplot = plt.imshow(img)
  plt.axis('off')
  plt.show()
labels[:5]
model.predict(xrays, batch_size=None, verbose=1, steps=None)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15239755884917175, 'fraction_numerical': 0.03016564952048823, 'mean_word_length': 6.709677419354839, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '136261', 'license': 'MIT', 'n_tokens_mistral': 2325, 'n_tokens_neox': 2167, 'n_words': 386, 'path': 'Predicting nodule on X ray.ipynb', 'repo_name': 'myfellowship/myfellowship', 'size': '5735'}