<jupyter_start><jupyter_text>converting the pandas dataframe into numpy arrays<jupyter_code>cluster1.iloc[:,-1].values
cluster1.iloc[:, :-1].values
cluster1.iloc[:,-1].values<jupyter_output><empty_output><jupyter_text>Checking The Values  For Cluster 1<jupyter_code>X_train_cluster1, X_test_cluster1, y_train_cluster1, y_test_cluster1 = train_test_split(cluster1.iloc[:, :-1].values, cluster1.iloc[:,-1].values, test_size=0.4)
X_train_cluster1
y_train_cluster1
X_test_cluster1
 y_test_cluster1
X_train_cluster2, X_test_cluster2, y_train_cluster2, y_test_cluster2 = train_test_split(cluster2.iloc[:, :-1].values, cluster2.iloc[:,-1].values, test_size=0.4)<jupyter_output><empty_output><jupyter_text>For Cluster 2<jupyter_code>X_train_cluster2
y_train_cluster2
X_test_cluster2
y_test_cluster2
No_of_test_cases_cluster1,No_of_test_cases_cluster2 = X_test_cluster1.shape[0],X_test_cluster2.shape[0]
print(No_of_test_cases_cluster1,"   ",No_of_test_cases_cluster2)
No_of_train_cluster1,No_of_train_cluster2 = X_train_cluster1.shape[0],X_train_cluster2.shape[0]
print(No_of_train_cluster1,"   ",No_of_train_cluster2)
No_of_columms_cluster1,No_of_columms_cluster2  = len(cluster1.columns),len(cluster2.columns)
print(No_of_columms_cluster1,"    ",No_of_columms_cluster2)
import math
def euclideanDistance(instance1, instance2, length):
	distance = 0
	distance = (instance1 - instance2)
	return abs(distance)
distances = []
import operator
clus1 = {"x_train":X_train_cluster1,"y_train":y_train_cluster1,
         "x_test":X_test_cluster1,"y_test":y_test_cluster1,"No_of_test_cases":No_of_test_cases_cluster1,
         "No_of_columms":No_of_columms_cluster1,"No_of_train":No_of_train_cluster1
        }
clus2 = {"x_train":X_train_cluster2,"y_train":y_train_cluster2,
         "x_test":X_test_cluster2,"y_test":y_test_cluster2,"No_of_test_cases":No_of_test_cases_cluster2,
         "No_of_columms":No_of_columms_cluster1,"No_of_train":No_of_train_cluster1
        }
cluster = {0:clus1,1:clus2}

cluster[0]
cluster[1]
# k = 3
# NeighboursSetDict1 = []
# # NeighboursSetDict2 = []

# for y in range(No_of_test_cases_cluster1):
#     NeighboursSet = []
#     for x in range(No_of_columms_cluster1-1):
        
#         print("x is",x)
#         distances = []
#         for i in range(No_of_train_cluster1):
#             print("x train is ",X_train_cluster1[i][x])
#             dist = euclideanDistance(X_train_cluster1[i][x],X_test_cluster1[y][x],1)
#             distances.append((X_train_cluster1[i][x], dist))
#             print("each distance ",distances)
#         distances.sort(key=operator.itemgetter(1))
#         print(distances ," heer")
#         neighbors = []
#         for x in range(k):
#             neighbors.append(distances[x][0])
#             print(neighbors)
#         NeighboursSet.append(neighbors)
        
#     NeighboursSetDict1.append(NeighboursSet)
# NeighboursSetDict1
cluster[1]["x_test"].shape[0]
k = 2
NeighboursSetDict1 = []
NeighboursSetDict2 = []
NeighboursSetDict = {0:NeighboursSetDict1,1:NeighboursSetDict2}
for _ in range(2):
    for y in range(cluster[_]["No_of_test_cases"]):
        print("Test Cases will be --->",cluster[_]["x_test"][y])
        NeighboursSet = []
        for x in range(cluster[_]["No_of_columms"]-1):

#             print("x is",x)
            distances = []
            for i in range(cluster[_]["No_of_train"]):
#                 print("x train is ",cluster[_]["x_train"][i][x])
                dist = euclideanDistance(cluster[_]["x_train"][i][x],cluster[_]["x_test"][y][x],1)
                distances.append((cluster[_]["x_train"][i][x], dist))
                print("  each distance for test data {0} and train data {1} is {2}".format(cluster[_]["x_test"][y][x],cluster[_]["x_train"][i][x],distances))
            distances.sort(key=operator.itemgetter(1))
            print("          sorted",distances)
            neighbors = []
            for x in range(k):
                neighbors.append(distances[x][0])
            print("      neighbours are",neighbors)
            NeighboursSet.append(neighbors)
        print("      NeighboursSet for Each test case",NeighboursSet)
        NeighboursSetDict[_].append(NeighboursSet)
    NeighboursSetDict[_]<jupyter_output>Test Cases will be ---> [2 9 9 9 9 9 9 9 9 9 9 9 9 9 9]
  each distance for test data 2 and train data 8 is [(8, 6)]
  each distance for test data 2 and train data 3 is [(8, 6), (3, 1)]
  each distance for test data 2 and train data 6 is [(8, 6), (3, 1), (6, 4)]
          sorted [(3, 1), (6, 4), (8, 6)]
      neighbours are [3, 6]
  each distance for test data 9 and train data 2 is [(2, 7)]
  each distance for test data 9 and train data 5 is [(2, 7), (5, 4)]
  each distance for test data 9 and train data 1 is [(2, 7), (5, 4), (1, 8)]
          sorted [(5, 4), (2, 7), (1, 8)]
      neighbours are [5, 2]
  each distance for test data 9 and train data 2 is [(2, 7)]
  each distance for test data 9 and train data 5 is [(2, 7), (5, 4)]
  each distance for test data 9 and train data 1 is [(2, 7), (5, 4), (1, 8)]
          sorted [(5, 4), (2, 7), (1, 8)]
      neighbours are [5, 2]
  each distance for test data 9 and train data 2 is [(2, 7)]
  each distance for test data 9 and train data 5 is [...]<jupyter_text>Here, k = 2,so every sublist in the list has two elements The values we will find by using function common will give the feature value of any row in the x_train,not the index to the row which knn find outs to be nearest .So ,if we found out the corresponding dependent variable of the row in which this feature value lies,we will get the result .<jupyter_code>NeighboursSetDict[0]
NeighboursSetDict[0][0]    #first index refer to cluster and then next index to test case
NeighboursSetDict[1]
# Function to find common elements in n arrays 
def commonElements(NeighboursSet): 
      
    # initialize result with first array as a set 
    result = set(NeighboursSet[0]) 
  
    # now iterate through list of arrays starting from 
    # second array and take intersection_update() of  
    # each array with result. Every operation will  
    # update value of result with common values in 
    # result set and intersected set 
    for currSet in NeighboursSet[1:]: 
        result.intersection_update(currSet) 
  
    return list(result)
common_for_cluster1_for_1st_test = commonElements(NeighboursSetDict[0][0])
common_for_cluster1_for_1st_test
common
common_for_cluster1_for_2nd_test = commonElements(NeighboursSetDict[0][1])
common_for_cluster1_for_2nd_test
final_common  = set(common_for_cluster1_for_1st_test) - (set(common_for_cluster1_for_1st_test) - set(common_for_cluster1_for_2nd_test))
final_common
common_for_cluster2_for_1st_test = commonElements(NeighboursSetDict[1][0])
common_for_cluster2_for_1st_test
common_for_cluster2_for_2nd_test = commonElements(NeighboursSetDict[1][1])
common_for_cluster2_for_2nd_test
final_common  = set(common_for_cluster2_for_1st_test) - (set(common_for_cluster2_for_1st_test) - set(common_for_cluster2_for_2nd_test))
final_common<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11520476122998441, 'fraction_numerical': 0.03684285106985971, 'mean_word_length': 4.208856088560886, 'pattern_counts': {'":': 14, '<': 16, '<?xml version=': 0, '>': 18, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 7, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '199004', 'license': 'MIT', 'n_tokens_mistral': 2632, 'n_tokens_neox': 2503, 'n_words': 570, 'path': 'Knn-IF Algorithm/clustering the dataset.ipynb', 'repo_name': 'jaikumardas577/projects', 'size': '7057'}