<jupyter_start><jupyter_text>Preprocessing  Data Reading & Inspection<jupyter_code>!git clone -l -s https://github.com/asyrofist/tugasPKPL.git cloned_pkpl
%cd cloned_pkpl
!ls
import pandas as pd
from time import time

print("Loading Dataset...")
t0 = time()
dataset2 = pd.read_csv('/content/cloned_pkpl/srs2.csv')
print("done in %0.3fs." % (time() - t0))
dataset2.head(n = 100)
import nltk
import string
from nltk.tokenize import word_tokenize 
from nltk.probability import FreqDist
from nltk.tokenize import sent_tokenize, word_tokenize
from nltk.corpus import stopwords
from nltk.stem import PorterStemmer
import matplotlib.pyplot as plt
import pandas as pd
import re 

nltk.download('stopwords')
nltk.download('punkt')

stemming = PorterStemmer()
stops = set(stopwords.words("english"))

# cleaning text
def text_cleaning(raw):
  
  # lowering word
  lower_case = raw.lower()

  # hapus punctuatioation & lower
  hasil_punctuation = lower_case.translate(str.maketrans("","",string.punctuation))
  
  # hapus whitespace
  hasil_whitespace = hasil_punctuation.strip()

  # hapus angka
  hasil_hapusangka = re.sub(r"\d+", "", hasil_whitespace)
  
  # tokenisasi 
  tokens = nltk.tokenize.word_tokenize(hasil_hapusangka)
  
  # Stemming
  stemmed_words = [stemming.stem(w) for w in tokens]
  
  # Remove stop words
  meaningful_words = [w for w in stemmed_words if not w in stops]
  
  # Rejoin meaningful stemmed words
  joined_words = ( " ".join(meaningful_words))
  
  # Return cleaned data
  return joined_words  

# applying
def apply_cleaning(hasil):
    cleaned_hasil = []
    for element in hasil:
        cleaned_hasil.append(text_cleaning(element))
    return cleaned_hasil

# Load data example (dari functional maupun nonfunctional)
statement = dataset2

# Truncate data for example
statement = statement.head(100)

# Get text to clean (dari row yang diinginkan)
text_to_clean = list(statement['Requirement Statement'])

# Clean text
print("Loading Original & Cleaned Text...")
t0 = time()
cleaned_text = apply_cleaning(text_to_clean)

# Show first example
print("done in %0.3fs." % (time() - t0))
pd.DataFrame([text_to_clean, cleaned_text],index=['ORIGINAL','CLEANED'], columns= statement['ID'])<jupyter_output>[nltk_data] Downloading package stopwords to /root/nltk_data...
[nltk_data]   Unzipping corpora/stopwords.zip.
[nltk_data] Downloading package punkt to /root/nltk_data...
[nltk_data]   Unzipping tokenizers/punkt.zip.
Loading Original & Cleaned Text...
done in 0.018s.<jupyter_text>Bag of Words  Implementing Bag of Words in scikit-learn<jupyter_code>import pandas as pd
from sklearn.feature_extraction.text import CountVectorizer
count_vector = CountVectorizer(cleaned_text)

print("Loading Counting Vector...")
t0 = time()

print(count_vector)
print("done in %0.3fs." % (time() - t0))
print("Loading feature names...")
t0 = time()

count_vector.fit(cleaned_text)
print(count_vector.get_feature_names())
print("done in %0.3fs." % (time() - t0))
print("Loading Document array...")
t0 = time()

doc_array = count_vector.transform(cleaned_text).toarray()
print(doc_array)
print("done in %0.3fs." % (time() - t0))
plt.hist(doc_array)
plt.xlabel('doc_array')
plt.ylabel('Frequency')
plt.show
print("Loading frequency matrix...")
t0 = time()

frequency_matrix = pd.DataFrame(doc_array,index=cleaned_text,columns=count_vector.get_feature_names())
print("done in %0.3fs." % (time() - t0))
frequency_matrix<jupyter_output>Loading frequency matrix...
done in 0.001s.<jupyter_text>Visualisasi<jupyter_code>import umap

print("Loading Visualisation...")
t0 = time()

visualisasi_bow = doc_array
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(visualisasi_bow)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors=None)
plt.xlabel('widht')
plt.ylabel('height')
plt.show()

print("done in %0.3fs." % (time() - t0))<jupyter_output>Loading Visualisation...<jupyter_text>Term Frequency - Inverse Document Frequency  Feature Generation using TF-IDF<jupyter_code>import numpy as np
import math

def l2_normalizer(vec):
    denom = np.sum([el**2 for el in vec])
    return [(el / math.sqrt(denom)) for el in vec]

doc_term_matrix_l2 = []
for vec in doc_array:
    doc_term_matrix_l2.append(l2_normalizer(vec))

print ('A regular old document term matrix: ') 
print (np.matrix(doc_array))
print ('\nA document term matrix with row-wise L2 norms of 1:')
print (np.matrix(doc_term_matrix_l2))
plt.hist(doc_term_matrix_l2)
plt.xlabel('term matrix l2')
plt.ylabel('Frequency')
plt.show
import string #allows for format()
    
def build_lexicon(corpus):
    lexicon = set()
    for doc in corpus:
        lexicon.update([word for word in doc.split()])
    return lexicon

def freq(term, document):
  return document.split().count(term)

def numDocsContaining(word, doclist):
    doccount = 0
    for doc in doclist:
        if freq(word, doc) > 0:
            doccount +=1
    return doccount 

def idf(word, doclist):
    n_samples = len(doclist)
    df = numDocsContaining(word, doclist)
    return np.log(n_samples / 1+df)

vocabulary = build_lexicon(cleaned_text)
mydoclist = cleaned_text

my_idf_vector = [idf(word, mydoclist) for word in vocabulary]

print ('Our vocabulary vector is [' + ', '.join(list(vocabulary)) + ']')
print ('The inverse document frequency vector is [' + ', '.join(format(freq, 'f') for freq in my_idf_vector) + ']')
import numpy as np

def build_idf_matrix(idf_vector):
    idf_mat = np.zeros((len(idf_vector), len(idf_vector)))
    np.fill_diagonal(idf_mat, idf_vector)
    return idf_mat

my_idf_matrix = build_idf_matrix(my_idf_vector)
my_idf_matrix
doc_term_matrix_tfidf = []

#performing tf-idf matrix multiplication
for tf_vector in doc_array:
    doc_term_matrix_tfidf.append(np.dot(tf_vector, my_idf_matrix))

#normalizing
doc_term_matrix_tfidf_l2 = []
for tf_vector in doc_term_matrix_tfidf:
    doc_term_matrix_tfidf_l2.append(l2_normalizer(tf_vector))
                                    
print (vocabulary)
print (np.matrix(doc_term_matrix_tfidf_l2))

# np.matrix() just to make it easier to look at
hasil_tfidf = np.matrix(doc_term_matrix_tfidf_l2)
print (hasil_tfidf)
plt.hist(hasil_tfidf)
plt.xlabel('tfidf')
plt.ylabel('Frequency')
plt.show
print("Loading frequency matrix...")
t0 = time()

frequency_TFIDF = pd.DataFrame(hasil_tfidf,index=cleaned_text,columns=count_vector.get_feature_names())
print("done in %0.3fs." % (time() - t0))
frequency_TFIDF<jupyter_output>Loading frequency matrix...
done in 0.001s.<jupyter_text>Visualisation<jupyter_code>import umap

print("Loading Topic Visualisation...")
t0 = time()

visual_tfidf = hasil_tfidf
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(visual_tfidf)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors=None)
plt.xlabel('widht')
plt.ylabel('height')
plt.show()

print("done in %0.3fs." % (time() - t0))<jupyter_output>Loading Topic Visualisation...<jupyter_text>Cosine SImilarity<jupyter_code>import numpy as np
from sklearn.metrics import pairwise_distances
from sklearn.metrics.pairwise import pairwise_kernels
X = np.array(visual_tfidf[0:])
Y = np.array(visual_tfidf)
print("X",X)
print("y",Y)
cosine_similaritas = pairwise_kernels(X, Y, metric='linear')
cosine_similaritas
plt.hist(cosine_similaritas)
plt.xlabel('cosine')
plt.ylabel('Frequency')
plt.show
print("Loading frequency matrix...")
t0 = time()

frequency_cosine = pd.DataFrame(cosine_similaritas, index=dataset2['ID'],columns=dataset2['ID'])
print("done in %0.3fs." % (time() - t0))
frequency_cosine<jupyter_output>Loading frequency matrix...
done in 0.001s.<jupyter_text>Visual Cosine<jupyter_code>import umap

print("Loading Topic Visualisation...")
t0 = time()

visual_cosine = cosine_similaritas
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(visual_cosine)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors=None)
plt.xlabel('widht')
plt.ylabel('height')
plt.show()

print("done in %0.3fs." % (time() - t0))<jupyter_output>Loading Topic Visualisation...<jupyter_text>K-means Clustering VSM (BOW, TFIDF, Cosine)<jupyter_code>import numpy as np
XVSM = np.array(hasil_tfidf)
yVSM = np.array(cosine_similaritas)
print("Loading Data X & Y...")
t0 = time()

print("X", XVSM, "\n")
print("Y", yVSM, "\n")

print("done in %0.3fs." % (time() - t0))
from sklearn.cluster import KMeans

print("Loading Kmeans...")
t0 = time()
kmeans = KMeans(n_clusters=2) # You want cluster the passenger records into 2: Survived or Not survived
print(kmeans.fit(XVSM))

print("done in %0.3fs." % (time() - t0))
print("Loading Data Correction...")
t0 = time()

correct = 0
for i in range(len(XVSM)):
    predict_me = np.array(XVSM[i].astype(float))
    predict_me = predict_me.reshape(-1, len(predict_me))
    prediction = kmeans.predict(predict_me)
    if prediction[0] == yVSM[i].all():
        correct += 1

print(correct/len(XVSM))

print("done in %0.3fs." % (time() - t0))
from sklearn.preprocessing import MinMaxScaler

print("Loading X Scaled...")
t0 = time()

scaler = MinMaxScaler()
XVSM_scaled = scaler.fit_transform(yVSM)

print(XVSM_scaled)

print("done in %0.3fs." % (time() - t0))<jupyter_output>Loading X Scaled...
[[1.         0.50541977 0.50541977 0.50541977 0.50015444 0.50541977
  0.57269296 0.50541977 0.67397514 0.45057401 0.29248764 0.22965476
  0.22915762 0.58255698 0.67249742 0.45255199 0.24864137 0.
  0.         0.13334684 0.39926629 0.         0.         0.
  0.15388796 0.        ]
 [0.50541977 1.         0.75270988 0.75270988 0.74486836 0.75270988
  0.57269296 0.75270988 0.45255199 0.45057401 0.29248764 0.22965476
  0.22915762 0.58255698 0.45155975 0.45255199 0.24864137 0.
  0.         0.13334684 0.17744164 0.         0.         0.
  0.15388796 0.        ]
 [0.50541977 0.75270988 1.         0.75270988 0.74486836 0.75270988
  0.57269296 0.75270988 0.45255199 0.45057401 0.29248764 0.22965476
  0.22915762 0.58255698 0.45155975 0.45255199 0.24864137 0.
  0.         0.13334684 0.17744164 0.         0.         0.
  0.15388796 0.        ]
 [0.50541977 0.75270988 0.75270988 1.         0.74486836 0.75270988
  0.57269296 0.75270988 0.45255199 0.45057401 0.29248764 0.22965476
 [...]<jupyter_text>Visual Kmeans Clustering<jupyter_code>import umap

print("Loading Topic Visualisation...")
t0 = time()

kmeans_VSM = XVSM_scaled
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(kmeans_VSM)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors='Blue')
plt.xlabel('widht')
plt.ylabel('height')
plt.show()

print("done in %0.3fs." % (time() - t0))<jupyter_output>Loading Topic Visualisation...<jupyter_text>Splitting data<jupyter_code># Import train_test_split function
from sklearn.model_selection import train_test_split

# Split dataset into training set and test set
X_train, X_test, y_train, y_test = train_test_split(X, kmeans_VSM, test_size=0.3,random_state=109) # 70% training and 30% test
X_train.shape, X_test.shape, y_train.shape, y_test.shape
y_train = np.argmax(y_train, axis=1)<jupyter_output><empty_output><jupyter_text>Model Classifer<jupyter_code>from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVC
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsClassifier

# Logistic Regression
clf = LogisticRegression(penalty='l2', C=0.1)
clf.fit(X_train, y_train)
y_predLR = clf.predict(X_test)

# Decision Tree
Dt = DecisionTreeClassifier()
Dt.fit(X_train, y_train)
y_predDT = Dt.predict(X_test)

# Decision Tree Regressor
Dtr = DecisionTreeRegressor()
Dtr.fit(X_train, y_train)
y_predDTR = Dtr.predict(X_test)

# Gaussian Naive Bias
gnb = GaussianNB()
gnb.fit(X_train, y_train)
y_predGNB = gnb.predict(X_test)

# random forest classifier
rfc = RandomForestClassifier()
rfc.fit(X_train, y_train)
y_predRFC = rfc.predict(X_test)

# random forest regressor
rfr = RandomForestRegressor(n_estimators = 100)
rfr.fit(X_train,y_train)
y_predRFR = rfr.predict(X_test)

# Support vector classifier
ppn = SVC(C=1, random_state = 0)
ppn.fit(X_train,y_train)
y_predSVC = ppn.predict(X_test)

# support vector regression
svm = SVR(C = 2, kernel = 'rbf', degree = 2)
svm.fit(X_train, y_train)
y_predSVR = svm.predict(X_test)

# k neearest neighbor classifier
Knn = KNeighborsClassifier(n_neighbors = 2, p =2, metric = 'minkowski')
Knn = Knn.fit(X_train,y_train)
y_predKNN = Knn.predict(X_test)
print("prediksi logistic regression", y_predLR)
print("prediksi Decision Tree", y_predDT)
print("prediksi Decision Tree Regression", y_predDTR)
print("prediksi Gaussian Naive Bias", y_predGNB)
print("prediksi random forest classifer", y_predRFC)
print("prediksi random forest regressor", y_predRFR)
print("prediksi support vector classifer", y_predSVC)
print("prediksi support vector regression", y_predSVR)
print("prediksi k nearest neighbor classifer", y_predKNN)<jupyter_output>prediksi logistic regression [ 0 24 11  0  3 21 25 19]
prediksi Decision Tree [ 0 24 11  0  6 11 11 11]
prediksi Decision Tree Regression [ 9. 24. 11.  9.  2. 24. 16. 19.]
prediksi Gaussian Naive Bias [ 0 24 11  0  1  0 19 19]
prediksi random forest classifer [ 0 24 11  0  3  4 19 19]
prediksi random forest regressor [ 9.59 19.74 11.29 10.13  3.   19.64 17.99 19.38]
prediksi support vector classifer [ 0 24 11  0  5  0 19 19]
prediksi support vector regression [ 9.87352192 13.20041381 11.14640452  9.86494133  7.39004161 11.91608436
 12.67811855 13.05675104]
prediksi k nearest neighbor classifer [ 0 19 10  0  2  0 19 19]<jupyter_text>Model Evaluation<jupyter_code># Python script for confusion matrix creation. 
from sklearn.metrics import confusion_matrix 
from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 

actual = [ 0, 24, 11,  0,  2, 0, 25, 19]
predicted = y_predSVC
results = confusion_matrix(actual, predicted) 

print ('Confusion Matrix :')
print(results,'\n') 
print ('Accuracy Score :',accuracy_score(actual, predicted)) 

print ('Report : ')
print (classification_report(actual, predicted))<jupyter_output>Confusion Matrix :
[[3 0 0 0 0 0 0]
 [0 0 1 0 0 0 0]
 [0 0 0 0 0 0 0]
 [0 0 0 1 0 0 0]
 [0 0 0 0 1 0 0]
 [0 0 0 0 0 1 0]
 [0 0 0 0 1 0 0]] 

Accuracy Score : 0.75
Report : 
              precision    recall  f1-score   support

           0       1.00      1.00      1.00         3
           2       0.00      0.00      0.00         1
           5       0.00      0.00      0.00         0
          11       1.00      1.00      1.00         1
          19       0.50      1.00      0.67         1
          24       1.00      1.00      1.00         1
          25       0.00      0.00      0.00         1

    accuracy                           0.75         8
   macro avg       0.50      0.57      0.52         8
weighted avg       0.69      0.75      0.71         8<jupyter_text>Latent Semantic Analysis  Document Term Matrix<jupyter_code>from time import time
from sklearn.feature_extraction.text import TfidfVectorizer

print("Loading Document matrix term matrix...")
t0 = time()

vectorizer = TfidfVectorizer(stop_words='english', 
max_features= 1000, # keep top 1000 terms 
max_df = 0.5, 
smooth_idf=True)
X = vectorizer.fit_transform(cleaned_text)
print(X.shape) # check shape of the document-term matrix
print("done in %0.3fs." % (time() - t0))
print(X)<jupyter_output>(0, 56)	0.7656187292181281
  (0, 18)	0.6432946148308866
  (1, 83)	0.8375737369732849
  (1, 3)	0.546324294840168
  (2, 90)	0.8375737369732849
  (2, 3)	0.546324294840168
  (3, 20)	0.8375737369732849
  (3, 3)	0.546324294840168
  (4, 41)	0.805768114051409
  (4, 3)	0.5922311595804763
  (5, 73)	0.805768114051409
  (5, 3)	0.5922311595804763
  (6, 72)	1.0
  (7, 61)	0.8375737369732849
  (7, 3)	0.546324294840168
  (8, 39)	0.5130898243327779
  (8, 66)	0.688097132919705
  (8, 18)	0.5130898243327779
  (9, 63)	0.4870826174343147
  (9, 37)	0.579702621453368
  (9, 76)	0.6532192545182897
  (10, 63)	0.6432946148308866
  (10, 72)	0.7656187292181281
  (11, 14)	0.5127301086919458
  (11, 10)	0.5777534324912074
  :	:
  (23, 28)	0.25597118360918264
  (23, 43)	0.22716287851035757
  (23, 49)	0.17791475565921835
  (23, 40)	0.1491064505603933
  (23, 12)	0.20672306075804345
  (24, 92)	0.3881763061742992
  (24, 5)	0.3881763061742992
  (24, 35)	0.3881763061742992
  (24, 1)	0.34448896097110665
  (24, 86)	0.28944941[...]<jupyter_text>Topic Modelling<jupyter_code>from sklearn.decomposition import TruncatedSVD

print("Loading Topic Modelling...")
t0 = time()

# SVD represent documents and terms in vectors 
svd_model = TruncatedSVD(n_components=20, algorithm='randomized', n_iter=100, random_state=122)
svd_model.fit(X)
print(len(svd_model.components_))
print("done in %0.3fs." % (time() - t0))
print(svd_model.components_)
plt.hist(svd_model.components_)
plt.xlabel('svd model components')
plt.ylabel('Frequency')
plt.show
print("Loading LSA Result...")
t0 = time()

terms = vectorizer.get_feature_names()
hasil_LSA = []
for i, comp in enumerate(svd_model.components_):
    terms_comp = zip(terms, comp)
    sorted_terms = sorted(terms_comp, key= lambda x:x[1], reverse=True)[:7]
    print("\nTopic "+str(i)+": ")
    # hasil_LSA.append(i)
    for t in sorted_terms:
        # print(t[0] + "\n")
        print(t[0])
        hasil_LSA.append(t)
print("done in %0.3fs." % (time() - t0))
print(hasil_LSA)
plt.hist(hasil_LSA)
plt.xlabel('Hasil LSA')
plt.ylabel('Frequency')
plt.show
# len(svd_model.components_)
# len(svd_model.fit_transform(X))<jupyter_output><empty_output><jupyter_text>Topic Visualisation<jupyter_code>import umap

print("Loading Topic Visualisation...")
t0 = time()

topic_lsa = svd_model.fit_transform(X)
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(topic_lsa)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors=None)
plt.title('Topic LSA VIsualisation')
plt.xlabel('widht')
plt.ylabel('height')
plt.show()

print("done in %0.3fs." % (time() - t0))
plt.hist(topic_lsa)
plt.xlabel('topic LSA')
plt.ylabel('Frequency')
plt.show<jupyter_output><empty_output><jupyter_text>Kmeans Clustering LSA<jupyter_code>import numpy as np
XLSA = np.array(svd_model.components_)
yLSA = np.array(topic_lsa)
print("Loading Data X & Y...")
t0 = time()

print("X", XLSA, "\n")
print("Y", yLSA, "\n")

print("done in %0.3fs." % (time() - t0))
from sklearn.cluster import KMeans

print("Loading Kmeans...")
t0 = time()
kmeans = KMeans(n_clusters=2) # You want cluster the passenger records into 2: Survived or Not survived
print(kmeans.fit(XLSA))

print("done in %0.3fs." % (time() - t0))
print("Loading Data Correction...")
t0 = time()

correct = 0
for i in range(len(XLSA)):
    predict_me = np.array(XLSA[i].astype(float))
    predict_me = predict_me.reshape(-1, len(predict_me))
    prediction = kmeans.predict(predict_me)
    if prediction[0] == yLSA[i].all():
        correct += 1

print(correct/len(XLSA))

print("done in %0.3fs." % (time() - t0))
from sklearn.preprocessing import MinMaxScaler


print("Loading X Scaled...")
t0 = time()

scaler = MinMaxScaler()
XLSA_scaled = scaler.fit_transform(XLSA)

print(XLSA_scaled)

print("done in %0.3fs." % (time() - t0))
print("Loading Correction...")
t0 = time()

print("X", XLSA, "\n")
print("X scaled", XLSA_scaled, "\n")

print("done in %0.3fs." % (time() - t0))
import umap

print("Loading Topic Visualisation...")
t0 = time()

kmeans_lsa = XLSA_scaled
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(kmeans_lsa)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors='Blue')
plt.title('Kmeans LSA Visualisation')
plt.xlabel('widht')
plt.ylabel('height')
plt.show()

print("done in %0.3fs." % (time() - t0))
plt.hist(kmeans_lsa)
plt.xlabel('KMeans LSA')
plt.ylabel('Frequency')
plt.show<jupyter_output><empty_output><jupyter_text>Splitting Data<jupyter_code># Import train_test_split function
from sklearn.model_selection import train_test_split

# Split dataset into training set and test set
X_train, X_test, y_train, y_test = train_test_split(XLSA, kmeans_lsa, test_size=0.3,random_state=109) # 70% training and 30% test
X_train.shape, X_test.shape, y_train.shape, y_test.shape
y_train = np.argmax(y_train, axis=1)<jupyter_output><empty_output><jupyter_text>Model Classifier<jupyter_code>from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVC
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsClassifier

# Logistic Regression
clf = LogisticRegression(penalty='l2', C=0.1)
clf.fit(X_train, y_train)
y_predLR = clf.predict(X_test)

# Decision Tree
Dt = DecisionTreeClassifier()
Dt.fit(X_train, y_train)
y_predDT = Dt.predict(X_test)

# Decision Tree Regressor
Dtr = DecisionTreeRegressor()
Dtr.fit(X_train, y_train)
y_predDTR = Dtr.predict(X_test)

# Gaussian Naive Bias
gnb = GaussianNB()
gnb.fit(X_train, y_train)
y_predGNB = gnb.predict(X_test)

# random forest classifier
rfc = RandomForestClassifier()
rfc.fit(X_train, y_train)
y_predRFC = rfc.predict(X_test)

# random forest regressor
rfr = RandomForestRegressor(n_estimators = 100)
rfr.fit(X_train,y_train)
y_predRFR = rfr.predict(X_test)

# Support vector classifier
ppn = SVC(C=1, random_state = 0)
ppn.fit(X_train,y_train)
y_predSVC = ppn.predict(X_test)

# support vector regression
svm = SVR(C = 2, kernel = 'rbf', degree = 2)
svm.fit(X_train, y_train)
y_predSVR = svm.predict(X_test)

# k neearest neighbor classifier
Knn = KNeighborsClassifier(n_neighbors = 2, p =2, metric = 'minkowski')
Knn = Knn.fit(X_train,y_train)
y_predKNN = Knn.predict(X_test)
print("prediksi logistic regression", y_predLR)
print("prediksi Decision Tree", y_predDT)
print("prediksi Decision Tree Regression", y_predDTR)
print("prediksi Gaussian Naive Bias", y_predGNB)
print("prediksi random forest classifer", y_predRFC)
print("prediksi random forest regressor", y_predRFR)
print("prediksi support vector classifer", y_predSVC)
print("prediksi support vector regression", y_predSVR)
print("prediksi k nearest neighbor classifer", y_predKNN)<jupyter_output>prediksi logistic regression [ 4 90 57 18  8 24]
prediksi Decision Tree [24 90 57  6 57 24]
prediksi Decision Tree Regression [20. 90. 20. 20. 20. 20.]
prediksi Gaussian Naive Bias [ 4  6  4 18 18  8]
prediksi random forest classifer [24 20 57  2 24  2]
prediksi random forest regressor [30.86 45.34 36.68 34.82 35.57 37.15]
prediksi support vector classifer [18  8  4  1  4  4]
prediksi support vector regression [13. 13. 13. 13. 13. 13.]
prediksi k nearest neighbor classifer [8 6 2 1 4 4]<jupyter_text>Model Evaluation<jupyter_code># Python script for confusion matrix creation. 
from sklearn.metrics import confusion_matrix 
from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 

# actual = [6, 36, 17, 10, 19, 19]
# actual = [4, 14, 12, 12, 8, 6]
actual = [4, 90, 20, 18, 8, 4]

# predicted = y_predLR
# predicted = y_predDT
# predicted = y_predDTR
# predicted = y_predGNB
# predicted = y_predRFC
# predicted = y_predRFR
predicted = y_predSVC
# predicted = y_predSVR
# predicted = y_predKNN

results = confusion_matrix(actual, predicted) 

print ('Confusion Matrix :')
print(results,'\n') 
print ('Accuracy Score :',accuracy_score(actual, predicted)) 

print ('Report : ')
print (classification_report(actual, predicted))<jupyter_output>Confusion Matrix :
[[0 0 0 0 0 0]
 [0 1 0 1 0 0]
 [0 1 0 0 0 0]
 [1 0 0 0 0 0]
 [0 1 0 0 0 0]
 [0 0 1 0 0 0]] 

Accuracy Score : 0.16666666666666666
Report : 
              precision    recall  f1-score   support

           1       0.00      0.00      0.00         0
           4       0.33      0.50      0.40         2
           8       0.00      0.00      0.00         1
          18       0.00      0.00      0.00         1
          20       0.00      0.00      0.00         1
          90       0.00      0.00      0.00         1

    accuracy                           0.17         6
   macro avg       0.06      0.08      0.07         6
weighted avg       0.11      0.17      0.13         6<jupyter_text>Latent Dirichlet Allocation  Import Library<jupyter_code>from time import time
from sklearn.feature_extraction.text import TfidfVectorizer, CountVectorizer
from sklearn.decomposition import NMF, LatentDirichletAllocation
from sklearn.datasets import fetch_20newsgroups<jupyter_output><empty_output><jupyter_text>Load dataset<jupyter_code>n_samples = 2000
n_features = 1000
n_components = 10
n_top_words = 20


def print_top_words(model, feature_names, n_top_words):
    for topic_idx, topic in enumerate(model.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])
        print(message)
    print()


# Load the 20 newsgroups dataset and vectorize it. We use a few heuristics
# to filter out useless terms early on: the posts are stripped of headers,
# footers and quoted replies, and common English words, words occurring in
# only one document or in at least 95% of the documents are removed.

print("Loading dataset...")
t0 = time()
data = cleaned_text
# data, _ = fetch_20newsgroups(shuffle=True, random_state=1,
#                              remove=('headers', 'footers', 'quotes'),
#                              return_X_y=True)
data_samples = data[:n_samples]
print(data_samples)
print("done in %0.3fs." % (time() - t0))
# Use tf-idf features for NMF.
print("Extracting tf-idf features for NMF...")
tfidf_vectorizer = TfidfVectorizer(max_df=0.95, min_df=2,
                                   max_features=n_features,
                                   stop_words='english')
t0 = time()
tfidf = tfidf_vectorizer.fit_transform(data_samples)
print(tfidf)
print("done in %0.3fs." % (time() - t0))
# Use tf (raw term count) features for LDA.
print("Extracting tf features for LDA...")
tf_vectorizer = CountVectorizer(max_df=0.95, min_df=2,
                                max_features=n_features,
                                stop_words='english')
t0 = time()
tf = tf_vectorizer.fit_transform(data_samples)
print(tf)
print("done in %0.3fs." % (time() - t0))
# Fit the NMF model
print("Fitting the NMF model (Frobenius norm) with tf-idf features, "
      "n_samples=%d and n_features=%d..."
      % (n_samples, n_features))
t0 = time()
nmf = NMF(n_components=n_components, random_state=1,
          alpha=.1, l1_ratio=.5).fit(tfidf)
print("done in %0.3fs." % (time() - t0))

print("\nTopics in NMF model (Frobenius norm):")
tfidf_feature_names = tfidf_vectorizer.get_feature_names()
print_top_words(nmf, tfidf_feature_names, n_top_words)
model_nmf = nmf
feature_names = tfidf_feature_names
n_top_words = n_top_words

for topic_idx, topic in enumerate(model_nmf.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])
        print(message)
plt.hist(model_nmf.components_)
plt.xlabel('Model NMF Components')
plt.ylabel('Frequency')
plt.show
plt.hist(model_nmf.fit_transform(tfidf_vectorizer.fit_transform(cleaned_text)))
plt.xlabel('model nmf fit transorm')
plt.ylabel('Frequency')
plt.show
# Fit the NMF model
print("Fitting the NMF model (generalized Kullback-Leibler divergence) with "
      "tf-idf features, n_samples=%d and n_features=%d..."
      % (n_samples, n_features))
t0 = time()
nmf = NMF(n_components=n_components, random_state=1,
          beta_loss='kullback-leibler', solver='mu', max_iter=1000, alpha=.1,
          l1_ratio=.5).fit(tfidf)
print("done in %0.3fs." % (time() - t0))

print("\nTopics in NMF model (generalized Kullback-Leibler divergence):")
tfidf_feature_names = tfidf_vectorizer.get_feature_names()
print_top_words(nmf, tfidf_feature_names, n_top_words)

print("Fitting LDA models with tf features, "
      "n_samples=%d and n_features=%d..."
      % (n_samples, n_features))
lda = LatentDirichletAllocation(n_components=n_components, max_iter=5,
                                learning_method='online',
                                learning_offset=50.,
                                random_state=0)
t0 = time()
lda.fit(tf)
print("done in %0.3fs." % (time() - t0))

print("\nTopics in LDA model:")
tf_feature_names = tf_vectorizer.get_feature_names()
print_top_words(lda, tf_feature_names, n_top_words)
model_lda = lda
feature_names = tfidf_feature_names
n_top_words = n_top_words

for topic_idx, topic in enumerate(model_lda.components_):
        message = "Topic #%d: " % topic_idx
        message += " ".join([feature_names[i]
                             for i in topic.argsort()[:-n_top_words - 1:-1]])
        print(message)
plt.hist(model_lda.components_)
plt.xlabel('Model LDA Components')
plt.ylabel('Frequency')
plt.show
plt.hist(model_lda.fit_transform(tfidf_vectorizer.fit_transform(cleaned_text)))
plt.xlabel('MOdel LDA Fit Transform')
plt.ylabel('Frequency')
plt.show
# len(model.components_)
# len(data_samples)
# len(model.fit_transform(vectorizer.fit_transform(cleaned_text)))
# len(model.fit_transform(vectorizer.fit_transform(model.components_)))
# len(model.fit_transform(vectorizer.fit_transform(data_samples)))

# len(nmf.components_)
# len(model.fit_transform(nmf.components_))<jupyter_output><empty_output><jupyter_text>Topic Visualisation<jupyter_code>import umap

print("Loading Topic Visualisation...")
t0 = time()

topic_lda = tf
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(topic_lda)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors=None)
plt.title('Topic LDA Visualization')
plt.xlabel('widht')
plt.ylabel('Height')
plt.show()

print("done in %0.3fs." % (time() - t0))<jupyter_output>Loading Topic Visualisation...<jupyter_text>Kmeans Clustering LDA<jupyter_code>import numpy as np
XLDA = np.array(model_lda.components_)
yLDA = np.array(model_lda.fit_transform(model_lda.components_))
print("Loading Data X & Y...")
t0 = time()

print("X", XLDA, "\n")
print("Y", yLDA, "\n")

print("done in %0.3fs." % (time() - t0))
from sklearn.cluster import KMeans

print("Loading Kmeans...")
t0 = time()
kmeans = KMeans(n_clusters=2) # You want cluster the passenger records into 2: Survived or Not survived
print(kmeans.fit(XLDA))

print("done in %0.3fs." % (time() - t0))
print("Loading Data Correction...")
t0 = time()

correct = 0
for i in range(len(XLDA)):
    predict_me = np.array(XLDA[i].astype(float))
    predict_me = predict_me.reshape(-1, len(predict_me))
    prediction = kmeans.predict(predict_me)
    if prediction[0] == yLDA[i].all():
        correct += 1

print(correct/len(XLDA))

print("done in %0.3fs." % (time() - t0))
from sklearn.preprocessing import MinMaxScaler


print("Loading X Scaled...")
t0 = time()

scaler = MinMaxScaler()
XLDA_scaled = scaler.fit_transform(XLDA)

print(XLDA_scaled)

print("done in %0.3fs." % (time() - t0))
print("Loading Correction...")
t0 = time()

print("X", XLDA, "\n")
print("X scaled", XLDA_scaled, "\n")

print("done in %0.3fs." % (time() - t0))
import umap

print("Loading Topic Visualisation...")
t0 = time()

kmeans_lda = XLDA_scaled
embedding = umap.UMAP(n_neighbors=3, min_dist=0.1, random_state=4).fit_transform(kmeans_lsa)
plt.figure(figsize=(7,5))
plt.scatter(embedding[:, 0], embedding[:, 1], c = None, s = 50 ,edgecolors='Blue')
plt.title("Kemans LDA Visualization")
plt.xlabel('widht')
plt.ylabel('height')
plt.show
plt.show()

print("done in %0.3fs." % (time() - t0))
plt.hist(kmeans_lda)
plt.xlabel('Kmeans LDA')
plt.ylabel('Frequency')
plt.show<jupyter_output><empty_output><jupyter_text>Splitting Data<jupyter_code># Import train_test_split function
from sklearn.model_selection import train_test_split

# Split dataset into training set and test set
X_train, X_test, y_train, y_test = train_test_split(XLDA, kmeans_lda, test_size=0.3,random_state=109) # 70% training and 30% test
X_train.shape, X_test.shape, y_train.shape, y_test.shape
print("X train", X_train)
print("X test", X_test)
print("y train", y_train)
print("X test", y_test)
y_train = np.argmax(y_train, axis=1)<jupyter_output><empty_output><jupyter_text>Model Classifier<jupyter_code>from sklearn.tree import DecisionTreeClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.tree import DecisionTreeRegressor
from sklearn.naive_bayes import GaussianNB
from sklearn.ensemble import RandomForestClassifier
from sklearn.ensemble import RandomForestRegressor
from sklearn.svm import SVC
from sklearn.svm import SVR
from sklearn.neighbors import KNeighborsClassifier

# Logistic Regression
clf = LogisticRegression(penalty='l2', C=0.1)
clf.fit(X_train, y_train)
y_predLR = clf.predict(X_test)

# Decision Tree
Dt = DecisionTreeClassifier()
Dt.fit(X_train, y_train)
y_predDT = Dt.predict(X_test)

# Decision Tree Regressor
Dtr = DecisionTreeRegressor()
Dtr.fit(X_train, y_train)
y_predDTR = Dtr.predict(X_test)

# Gaussian Naive Bias
gnb = GaussianNB()
gnb.fit(X_train, y_train)
y_predGNB = gnb.predict(X_test)

# random forest classifier
rfc = RandomForestClassifier()
rfc.fit(X_train, y_train)
y_predRFC = rfc.predict(X_test)

# random forest regressor
rfr = RandomForestRegressor(n_estimators = 100)
rfr.fit(X_train,y_train)
y_predRFR = rfr.predict(X_test)

# Support vector classifier
ppn = SVC(C=1, random_state = 0)
ppn.fit(X_train,y_train)
y_predSVC = ppn.predict(X_test)

# support vector regression
svm = SVR(C = 2, kernel = 'rbf', degree = 2)
svm.fit(X_train, y_train)
y_predSVR = svm.predict(X_test)

# k neearest neighbor classifier
Knn = KNeighborsClassifier(n_neighbors = 2, p =2, metric = 'minkowski')
Knn = Knn.fit(X_train,y_train)
y_predKNN = Knn.predict(X_test)
print("prediksi logistic regression", y_predLR)
print("prediksi Decision Tree", y_predDT)
print("prediksi Decision Tree Regression", y_predDTR)
print("prediksi Gaussian Naive Bias", y_predGNB)
print("prediksi random forest classifer", y_predRFC)
print("prediksi random forest regressor", y_predRFR)
print("prediksi support vector classifer", y_predSVC)
print("prediksi support vector regression", y_predSVR)
print("prediksi k nearest neighbor classifer", y_predKNN)<jupyter_output>prediksi logistic regression [22 22 22]
prediksi Decision Tree [22  9  1]
prediksi Decision Tree Regression [23. 18. 22.]
prediksi Gaussian Naive Bias [22 22 22]
prediksi random forest classifer [22 22 23]
prediksi random forest regressor [17.94 15.85 15.72]
prediksi support vector classifer [22 22 22]
prediksi support vector regression [18.60724939 17.45024725 18.76016895]
prediksi k nearest neighbor classifer [ 9 22 22]<jupyter_text>Model Evaluation<jupyter_code># Python script for confusion matrix creation. 
from sklearn.metrics import confusion_matrix 
from sklearn.metrics import accuracy_score 
from sklearn.metrics import classification_report 

# actual = [3, 0, 1]
# actual = [2 , 0, 0]
actual = [0 , 0, 0]

# predicted = y_predLR
# predicted = y_predDT
# predicted = y_predDTR
# predicted = y_predGNB
# predicted = y_predRFC
# predicted = y_predSVC
predicted = y_predKNN
results = confusion_matrix(actual, predicted) 

print ('Confusion Matrix :')
print(results,'\n') 
print ('Accuracy Score :',accuracy_score(actual, predicted)) 

print ('Report : ')
print (classification_report(actual, predicted))<jupyter_output>Confusion Matrix :
[[0 1 2]
 [0 0 0]
 [0 0 0]] 

Accuracy Score : 0.0
Report : 
              precision    recall  f1-score   support

           0       0.00      0.00      0.00       3.0
           9       0.00      0.00      0.00       0.0
          22       0.00      0.00      0.00       0.0

    accuracy                           0.00       3.0
   macro avg       0.00      0.00      0.00       3.0
weighted avg       0.00      0.00      0.00       3.0
{'fraction_non_alphanumeric': 0.12357790601813685, 'fraction_numerical': 0.07034899697719153, 'mean_word_length': 4.219592656339644, 'pattern_counts': {'":': 1, '<': 88, '<?xml version=': 0, '>': 89, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 34, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '26', 'dir': 'jupyter-structured-clean-dedup', 'id': '525035', 'license': 'MIT', 'n_tokens_mistral': 15373, 'n_tokens_neox': 13571, 'n_words': 3193, 'path': 'notebook/traceability.ipynb', 'repo_name': 'asyrofist/tracereq', 'size': '36390'}