<jupyter_start><jupyter_text><NAME>, 17PH20006  Basics of cluster analysis  Import the relevant libraries<jupyter_code>!pip install ijson==2.2
#other packages comes with anaconda distribution
import pandas as pd
import numpy as np
import matplotlib.pyplot as plt
import seaborn as sns
import ijson
import json

from pandas.io.json import json_normalize
sns.set()
from sklearn.cluster import KMeans


import re
import sys
import json
import re, string
import copy
from nltk.corpus import stopwords
import string
from string import punctuation
import random
from tqdm.notebook import tqdm

#sklearn 0.24.1
#nltk 3.2.4<jupyter_output><empty_output><jupyter_text>Load the data<jupyter_code># open json file
with open('Tweets.json','r') as file:
    data = json.load(file)

# json dictionary
#print(type(data))
# loading into a DataFrame
df_json = pd.DataFrame(data)
df_json.head()<jupyter_output><empty_output><jupyter_text>1.  removing hashtags and cleaning up<jupyter_code># remove the hashtags, mentions and unwanted characters.
def clean_text(df, text_field):



    df[text_field] = df[text_field].str.lower()
    df[text_field] = df[text_field].apply(lambda elem: re.sub(r' (@[A-Za-z0–9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)|^rt|http.+?', "", elem)) 
    
    return df

df_json = clean_text(df_json, 'text')
df_json.head()<jupyter_output><empty_output><jupyter_text>Kmeans class with jccard distance, Silhouette methode<jupyter_code>regex = re.compile('[%s]' % re.escape(string.punctuation))
cachedStopWords = stopwords.words('english')

class kMeans():
    def __init__(self, seeds, tweets):
        self.seeds = seeds
        self.tweets = tweets
        self.max_iterations = 1000
        self.k = len(seeds)

        self.clusters = {} # cluster to tweetID
        self.rev_clusters = {} # reverse index, tweetID to cluster
        self.jaccardMatrix = {} # stores pairwise jaccard distance in a matrix

        self.initializeClusters()
        self.initializeMatrix()
        
    def jaccardDistance(self, setA, setB):
        # Calcualtes the Jaccard Distance of two sets
        try:
            return 1 - float(len(setA.intersection(setB))) / float(len(setA.union(setB)))
        except TypeError:
            print ('Invalid type. Type set expected.')
    def bagOfWords(self, string):
        # Returns a bag of words from a given string
        # Space delimited, removes punctuation, lowercase
        # Cleans text from url, stop words, tweet @, and 'rt'
        words = string.lower().strip().split(' ')
        for word in words:
            word = word.rstrip().lstrip()
            if not re.match(r'^https?:\/\/.*[\r\n]*', word) \
            and not re.match('^@.*', word) \
            and not re.match('\s', word) \
            and word not in cachedStopWords \
            and word != 'rt' \
            and word != '':
                yield regex.sub('', word)
                
    def initializeMatrix(self):
        # Dynamic Programming: creates matrix storing pairwise jaccard distances
        for ID1 in self.tweets:
            self.jaccardMatrix[ID1] = {}
            bag1 = set(self.bagOfWords(self.tweets[ID1]))
            for ID2 in self.tweets:
                if ID2 not in self.jaccardMatrix:
                    self.jaccardMatrix[ID2] = {}
                bag2 = set(self.bagOfWords(self.tweets[ID2]))
                distance = self.jaccardDistance(bag1, bag2)
                self.jaccardMatrix[ID1][ID2] = distance
                self.jaccardMatrix[ID2][ID1] = distance
                
    def initializeClusters(self):
        # Initialize tweets to no cluster
        for ID in self.tweets:
            self.rev_clusters[ID] = -1

        # Initialize clusters with seeds
        for k in range(self.k):
            self.clusters[k] = set([self.seeds[k]])
            self.rev_clusters[self.seeds[k]] = k
            
    def calcNewClusters(self):
        # Initialize new cluster
        new_clusters = {}
        new_rev_cluster = {}
        loss_total=0
        for k in range(self.k):
            new_clusters[k] = set()

        for ID in self.tweets:
            min_dist = float("inf")
            min_dist1 = float("inf")
            min_cluster = self.rev_clusters[ID]

            # Calculate min average distance to each cluster
            cnt=0
            for k in self.clusters:
                dist = 0
                count = 0
                cnt+=1
                for ID2 in self.clusters[k]:
                    dist += self.jaccardMatrix[ID][ID2]
                    count += 1
                if count > 0:
                    avg_dist = dist/float(count)
                    if min_dist > avg_dist:
                        min_dist = avg_dist
                        min_cluster = k
                        
            for k in self.clusters:
                if(k==min_cluster):
                    continue
                dist = 0
                count = 0
                cnt+=1
                for ID2 in self.clusters[k]:
                    dist += self.jaccardMatrix[ID][ID2]
                    count += 1
                if count > 0:
                    avg_dist1 = dist/float(count)
                    if min_dist1 > avg_dist1:
                        min_dist1 = avg_dist1
                        min_prev = k
                        
                        
                        
            a = 0
            b = 0
            count = 0  

            
            for ID2 in self.clusters[min_cluster]:
                a += self.jaccardMatrix[ID][ID2]
                count +=1
            
            for ID2 in self.clusters[min_prev]:
                b += self.jaccardMatrix[ID][ID2]

            if(count==1):
                loss_total+=0
            else:
                
                loss_total+=(b-a)/max(a,b)
            new_clusters[min_cluster].add(ID)
            new_rev_cluster[ID] = min_cluster
        
        loss_total=loss_total/len(self.tweets)
        
        return new_clusters, new_rev_cluster, loss_total
    
    
    def converge(self):
        # Initialize previous cluster to compare changes with new clustering
        new_clusters, new_rev_clusters, _ = self.calcNewClusters()
        self.clusters = copy.deepcopy(new_clusters)
        self.rev_clusters = copy.deepcopy(new_rev_clusters)

        # Converges until old and new iterations are the same
        iterations = 1
        while iterations < self.max_iterations:
            new_clusters, new_rev_clusters,_ = self.calcNewClusters()
            iterations += 1
            if self.rev_clusters != new_rev_clusters:
                self.clusters = copy.deepcopy(new_clusters)
                self.rev_clusters = copy.deepcopy(new_rev_clusters)
            else:
                #print iterations
                return
            
    def printClusterText(self):
        # Prints text of clusters
        for k in self.clusters:
            for ID in self.clusters[k]:
                print(self.tweets[ID])
            print ('\n')
 
    def printClusters(self):
        # Prints cluster ID and tweet IDs for that cluster
        for k in self.clusters:
            print (str(k) + ':' + ','.join(map(str,self.clusters[k])))

    def printMatrix(self):
        # Prints jaccard distance matrix
        for ID in self.tweets:
            for ID2 in self.tweets:
                print (ID, ID2, self.jaccardMatrix[ID][ID2])
                
    
tweets={}
for i in range(len(df_json['id'])):
    temp = df_json['text'][i]
    temp = temp.replace("'","")
    temp = temp.replace("[","")
    temp = temp.replace("]","")
    df_json['text'][i]=temp
    
    
    tweets[df_json['id'][i]] = temp

    
def loss_kmeans(tweets=tweets,k=24,df_json=df_json):
    
    ids=list(df_json['id'])
    random.shuffle(ids)
    seeds=ids[0:k]
    kmeans = kMeans(seeds, tweets)
    kmeans.converge()
    result=kmeans.clusters
    dist=kmeans.jaccardMatrix
    _,_,square_dist=kmeans.calcNewClusters()

    total=0

    for i in result:
        for k in result:
            if(i!=k):
                continue
            temp=[]
            for j in result[i]:
                for l in result[k]:
                    #print(i,j,k,l)
                    #print(dist[j][l])
                    #temp.append(dist[j][l])
                    total=total+dist[j][l]**2
            
            #if(len(temp)>0):
            #    total=total+max(temp)**2 
    
            
    total=total/2
    
    return square_dist
loss=[]
cluster_size=[]
for i in tqdm(range(3,30)):
    loss.append(loss_kmeans(tweets=tweets,k=i,df_json=df_json))
    cluster_size.append(i+1)
plt.plot(cluster_size,loss)
plt.xlabel('Number of clusters')
plt.ylabel('Loss')<jupyter_output><empty_output><jupyter_text>Taking k=24<jupyter_code>k=24
ids=list(df_json['id'])
random.shuffle(ids)
seeds=ids[0:k]
kmeans = kMeans(seeds, tweets)
kmeans.converge()
result=kmeans.clusters



col=[]
numbers=[]
for i in result:
    print("Cluster:  -->",i)
    print('-'*100)
    col.append("cluster "+str(i))
    cnt=0
    for j in result[i]:
        cnt+=1
        print(j)
        print(tweets[j])
    
    numbers.append(cnt)
df=pd.DataFrame([],columns=col)
df_length = len(df)
df.loc[df_length] = numbers
df
fig = plt.figure(figsize =(10, 7)) 
#a = new_groups_df.drop(['total'], axis = 1)
plt.pie(df.loc[df_length], labels = df.columns)
plt.title('A pie chart showing the volumes of tweets under different categories.')
plt.show()<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11070149568148305, 'fraction_numerical': 0.011059616599957868, 'mean_word_length': 1.9001221747098351, 'pattern_counts': {'":': 0, '<': 23, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '278886', 'license': 'CC-BY-3.0', 'n_tokens_mistral': 3118, 'n_tokens_neox': 2951, 'n_words': 670, 'path': '17PH20006/K-means.ipynb', 'repo_name': 'ansumandas441/ansumandas441.github.io', 'size': '9499'}