<jupyter_start><jupyter_text><jupyter_code>import numpy as np
import pandas as pd
import seaborn as sns
import matplotlib.pyplot as plt
sns.set()
import h5py 
from tqdm import tqdm_notebook as tqdm
import random
from sklearn import preprocessing

from google.colab import drive
drive.mount('/content/drive')
df = pd.read_csv('/content/drive/MyDrive/Fabric Detection/Textile Defect detection/train32.csv')
import h5py
df.head()
f = h5py.File('/content/drive/MyDrive/Fabric Detection/Textile Defect detection/matchingtDATASET_test_32.h5', 'r')
f.keys()
data=f['hole']['angle0']
plt.imshow(data[0][0],cmap='gray')
f.close()
def getData(path):
    data=[]
    f=h5py.File(path, 'r')
    for label in tqdm(f.keys()):
        for angleKey in f[label].keys():
            for arr in f[label][angleKey]:
                data.append([arr[0],angleKey,label])
    f.close()
    return (data)
train_data=getData('/content/drive/MyDrive/Fabric Detection/Textile Defect detection/matchingtDATASET_train_32.h5')
test_data=getData('/content/drive/MyDrive/Fabric Detection/Textile Defect detection/matchingtDATASET_test_32.h5')<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:4: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
  after removing the cwd from sys.path.<jupyter_text>A Jupyter widget could not be displayed because the widget state could not be found. This could happen if the kernel storing the widget is no longer available, or if the widget state was not saved in the notebook. You may be able to create the widget by running the appropriate cells.A Jupyter widget could not be displayed because the widget state could not be found. This could happen if the kernel storing the widget is no longer available, or if the widget state was not saved in the notebook. You may be able to create the widget by running the appropriate cells.<jupyter_code>LE=preprocessing.LabelEncoder()
def splitData(data):
    random.shuffle(data)
    X=[]
    angleArr=[]
    y=[]
    for i in tqdm(data):
        image=i[0]
        angle=i[1]
        label=i[2]
        X.append(image.flatten())
        angleArr.append(angle)
        y.append(label)
    y=LE.fit_transform(y)
    return(X,y,angleArr)
X_train,y_train,angles_train=splitData(train_data)
X_test,y_test,angles_test=splitData(test_data)<jupyter_output>/usr/local/lib/python3.7/dist-packages/ipykernel_launcher.py:7: TqdmDeprecationWarning: This function will be removed in tqdm==5.0.0
Please use `tqdm.notebook.tqdm` instead of `tqdm.tqdm_notebook`
  import sys<jupyter_text>A Jupyter widget could not be displayed because the widget state could not be found. This could happen if the kernel storing the widget is no longer available, or if the widget state was not saved in the notebook. You may be able to create the widget by running the appropriate cells.A Jupyter widget could not be displayed because the widget state could not be found. This could happen if the kernel storing the widget is no longer available, or if the widget state was not saved in the notebook. You may be able to create the widget by running the appropriate cells.<jupyter_code>from sklearn.svm import SVC
clf = SVC(kernel='rbf')
 import tensorflow as tf 
 tf.test.gpu_device_name()
%tensorflow_version 2.x
import tensorflow as tf
import timeit

device_name = tf.test.gpu_device_name()
if device_name != '/device:GPU:0':
  print(
      '\n\nThis error most likely means that this notebook is not '
      'configured to use a GPU.  Change this in Notebook Settings via the '
      'command palette (cmd/ctrl-shift-P) or the Edit menu.\n\n')
  raise SystemError('GPU device not found')

def cpu():
  with tf.device('/cpu:0'):
    random_image_cpu = tf.random.normal((100, 100, 100, 3))
    net_cpu = tf.keras.layers.Conv2D(32, 7)(random_image_cpu)
    return tf.math.reduce_sum(net_cpu)

def gpu():
  with tf.device('/device:GPU:0'):
    random_image_gpu = tf.random.normal((100, 100, 100, 3))
    net_gpu = tf.keras.layers.Conv2D(32, 7)(random_image_gpu)
    return tf.math.reduce_sum(net_gpu)
  
# We run each op once to warm up; see: https://stackoverflow.com/a/45067900
cpu()
gpu()

# Run the op several times.
print('Time (s) to convolve 32x7x7x3 filter over random 100x100x100x3 images '
      '(batch x height x width x channel). Sum of ten runs.')
print('CPU (s):')
cpu_time = timeit.timeit('cpu()', number=10, setup="from __main__ import cpu")
print(cpu_time)
print('GPU (s):')
gpu_time = timeit.timeit('gpu()', number=10, setup="from __main__ import gpu")
print(gpu_time)
print('GPU speedup over CPU: {}x'.format(int(cpu_time/gpu_time)))
from tensorflow.python.client import device_lib
device_lib.list_local_devices()
clf.fit(X_train,y_train)
y_predict = clf.predict(X_test)
from sklearn.metrics import classification_report
print(classification_report(y_test,y_predict))
#display all the columns in the data set
pd.set_option('display.max_columns',None)
df.head()
df.info()
sns.pairplot(df,hue='indication_value')
df.shape
#check the null values
df.isnull().sum()
#check the count values
df.isnull().count()
#seems that there is no null values
#check the dependent features output values
df['indication_value'].value_counts()
##Independent and dependent features
X=df.drop("indication_value",axis=1)
y=df.indication_value
X["indication_type"].value_counts()
X = X.replace({"color": 0,"cut":1,"thread":2,"hole":3,"metal_contamination":4,"good":5})
X.head()
X.drop("split", axis=1,inplace=True)
y.head()
from sklearn.model_selection import train_test_split
X_train, X_test, y_train, y_test = train_test_split(X,y,test_size=0.3,random_state=0)
X_train.corr()
# #using Pearson Correlation(ranges between -1 to +1)
# plt.figure(figsize=(12,10))
# cor = X_train.corr()
# sns.heatmap(cor, annot=True,cmap=plt.cm.CMRmap_r)
# plt.show()
# # with the following function we can select highly correlated features
# # it will remove the first feature that is correlated with anything other feature

# def correlation(dataset, threshold):
#     col_corr = set()  # Set of all the names of correlated columns
#     corr_matrix = dataset.corr()
#     for i in range(len(corr_matrix.columns)):
#         for j in range(i):
#             if abs(corr_matrix.iloc[i, j]) > threshold: # we are interested in absolute coeff value
#                 colname = corr_matrix.columns[i]  # getting the name of column
#                 col_corr.add(colname)
#     return col_corr
# #abs is use to take both positive and negative values
# corr_features = correlation(X_train, 0.8)
# len(set(corr_features))
#Find Corelation between dependent and independent Features

plt.figure(figsize=(18,18))
sns.heatmap(df.corr(), annot=True, cmap = plt.cm.bone,)
plt.show()
plt.figure(figsize=(28,28))
sns.clustermap(df.corr(), annot=True, cmap="gist_ncar_r")
plt.show()
from sklearn.ensemble import ExtraTreesRegressor
# Important feature using ExtraTreesRegressor

from sklearn.ensemble import ExtraTreesRegressor
selection = ExtraTreesRegressor()
selection.fit(X_train, y_train)
plt.figure(figsize= (12,8))
feat_importance = pd.Series(selection.feature_importances_, index=X.columns)
feat_importance.nlargest(20).plot(kind='barh')
plt.show()

from sklearn.ensemble import RandomForestRegressor
from sklearn.datasets import make_classification
reg_rf = RandomForestRegressor()
reg_rf.fit(X_train,y_train)
y_pred = reg_rf.predict(X_test)
reg_rf.score(X_train,y_train)
reg_rf.score(X_test,y_test)
plt.figure(figsize=(9,9))
sns.distplot(y_test-y_pred,color='r')
plt.show()
plt.figure(figsize=(9,9))
plt.scatter(y_test,y_pred, alpha=0.5,cmap="r",edgecolors="r")
plt.xlabel("y_test")
plt.ylabel("y_pred")
plt.show()
from sklearn import metrics
print('MAE',metrics.mean_absolute_error(y_test,y_pred))
print('MSE',metrics.mean_squared_error(y_test,y_pred))
print('RMSE',np.sqrt(metrics.mean_squared_error(y_test,y_pred)))

plt.figure(figsize=(9,9))
plt.plot(y_test,y_pred)
plt.plot(metrics.mean_squared_error(y_test,y_pred))
plt.plot('RMSE',np.sqrt(metrics.mean_squared_error(y_test,y_pred),))
# plt.plot('MAE',metrics.mean_absolute_error(y_test,y_pred))
#plt.tight_layout()
plt.show()
metrics.r2_score(y_test, y_pred)
# Instantiation 
import xgboost as xg
xgb_r = xg.XGBRegressor(objective ='reg:linear', 
                  n_estimators = 10, seed = 123) 
# Fitting the model 
xgb_r.fit(X_train, y_train) 
  
# Predict the model 
pred = xgb_r.predict(X_test) 
  

plt.figure(figsize=(9,9))
plt.scatter(y_test,pred,edgecolors='g')
from sklearn.metrics import roc_auc_score
from sklearn.datasets import load_breast_cancer
from sklearn.linear_model import LogisticRegression
from sklearn.datasets import load_iris
X, y = load_iris(return_X_y=True)
clf = LogisticRegression(solver="liblinear").fit(X, y)
roc_auc_score(y, clf.predict_proba(X), multi_class='ovr')
#print('ROC Accuracy', roc_auc_score(y_test,pred))
clf
plt.figure(figsize=(9,9))
plt.scatter(y_test,pred,edgecolors='g')
import numpy as np
import matplotlib.pyplot as plt
from sklearn.linear_model import LinearRegression
linreg = LinearRegression()
linreg.fit(X,y)
# fitting naive bayes to the training set 
from sklearn.naive_bayes import GaussianNB 
from sklearn.metrics import confusion_matrix 
  
classifier = GaussianNB(); 
classifier.fit(X_train, y_train) 
  
# predicting test set results 
y_pred = classifier.predict(X_test) 
  
# making the confusion matrix 
cm = confusion_matrix(y_test, y_pred) 
cm
plt.figure(figsize=(9,9))
sns.clustermap(cm, annot=True, fmt=".3f", linewidths=.5, square = True, cmap = 'Blues_r');
plt.ylabel('Actual label');
plt.xlabel('Predicted label');
all_sample_title = 'Accuracy Score: {0}'.format(score)
plt.title(all_sample_title, size = 15);
plt.figure(figsize=(9,9))
sns.heatmap(cm, annot=True, fmt=".3f", linewidths=.5, square = True, cmap = 'Blues_r');
plt.ylabel('Actual label');
plt.xlabel('Predicted label');
all_sample_title = 'Accuracy Score: {0}'.format(score)
plt.title(all_sample_title, size = 15);




from sklearn.metrics import accuracy_score
accuracy_score(y_test,y_pred)
plt.scatter(y_test,y_pred,linewidths=-2,cmap="fre",edgecolors='g')
from sklearn.neighbors import KNeighborsClassifier
from sklearn.metrics import accuracy_score
from sklearn.metrics import confusion_matrix
from sklearn.metrics import plot_confusion_matrix
knn_clf = KNeighborsClassifier(n_neighbors=9)
knn_clf.fit(X_train,y_train)
knn_predictions = knn_clf.predict(X_test)
print('Accuracy of the knn algorithm is : {}'.format(accuracy_score(y_test,knn_predictions)))
plt.scatter(y_test,knn_predictions,edgecolors='r')
confusion_matrix(y_test, knn_predictions)





from sklearn import linear_model
from sklearn.metrics import mean_squared_error, r2_score
model = linear_model.LinearRegression()
#Bulid training model
model.fit(X_train,y_train)
#Apply trained model to make prediction(on test data set)
y_predict = model.predict(X_test)
print("Coefficients:",model.coef_)
print("Intercept:",model.intercept_)
print("Mean Squared error(MSE):%.2f"
     % mean_squared_error(y_test,y_predict))
print('Cofficient of determination (R^2): %.2f'
     %r2_score(y_test, y_predict))
#stream Formating
r2_score(y_test,y_predict)
r2_score(y_test,y_predict).dtype
y_test.array
y_predict
sns.scatterplot(y_test, y_predict)
sns.scatterplot(y_test, y_predict, marker="+",alpha=1)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11790047476701249, 'fraction_numerical': 0.014682609460172323, 'mean_word_length': 5.703005303476724, 'pattern_counts': {'":': 6, '<': 11, '<?xml version=': 0, '>': 12, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '531833', 'license': 'MIT', 'n_tokens_mistral': 4158, 'n_tokens_neox': 3953, 'n_words': 971, 'path': 'Fabric_Detection.ipynb', 'repo_name': 'himanshusharma9034/Fabric-Defect-Detection-', 'size': '11374'}