<jupyter_start><jupyter_text>Debug and games of the block class<jupyter_code>test_block = CreateBlocks(transcripts_jsons,window_size=40)
#test = test_block.partion_by_sliding_windows(120, 20)
cr,inde = test_block.partion()
print(inde)
print(len(test_block.tokenized_corpus))
print(len(test_block.word_timestamp))
print(len(test_block.tokenized_corpus))
print(len(test_block.block_sizes))
cr_2,inde_2 = test_block.partion_by_sliding_windows(120,20)
print(inde_2)
test_block.get_block_timestamp()
from abc import ABC
#import gensim.corpora as corpora
#import gensim
import numpy as np



class CalculateVectors(ABC):
    '''
        2.	Calculate vector 
            a.	Tf-idf , TF, Word embedding 
            b.	Make the calculation across all text (like word embedding) or
                just within the block or adjacent block (what ever you would like)
                Choose sliding window – fixed size or documents
    '''
    
    blocks = NotImplemented
    
    def calc(self):
        pass


    
'''
    consider aggregate the 2 classes into single class
'''    
    
from sklearn.feature_extraction.text import CountVectorizer
class TFCalculator(CalculateVectors):
    def __init__(self,blocks): 
        self.blocks = blocks
        self.vectorizer = CountVectorizer()
        
    
    # each vector is n word in all of the vocabulary
    def calc(self):
        #print(self.blocks)
        con_blocks = [' '.join(blk) for blk in self.blocks]
        #print(con_blocks)
        return self.vectorizer.fit_transform(con_blocks).toarray()
        
from sklearn.feature_extraction.text import TfidfVectorizer
class TFIDFCalculator(CalculateVectors):
    def __init__(self,blocks):
        self.blocks = blocks
        self.vectorizer = TfidfVectorizer()
    
    def calc(self):
        con_blocks = [' '.join(blk) for blk in self.blocks]
        return self.vectorizer.fit_transform(con_blocks).toarray()

from abc import ABC
from sklearn.metrics.pairwise import cosine_similarity


class SimilarityCalc(ABC):
    
    vectors = NotImplemented
    
    def calc(self):
        pass



class DirectSimilarity(SimilarityCalc):
    '''    
        This class is used to calculate the direct similarity between 2 vectors
    '''
    
    def __init__(self,vectors):
        self.vectors = vectors
        self.similarity_function = cosine_similarity# consider using other similarity function than cosine 
        
    # notice it is more valid of vectors like the text tiling looking for minima example
    def calc(self):
        return [self.similarity_function(self.vectors[i].reshape(1,-1),\
                                         self.vectors[i+1].reshape(1,-1))[0][0]\
                for i in range(0,len(self.vectors) - 1)]
        #pass
    
    
class Clustering():
    pass

from numpy import diff,sign
class TextTilingClustering():
    
    '''
        used to find clusters by local minima
    '''
    @staticmethod
    def find_cluster(similarities_array,gap_times):
        minimas = (diff(sign(diff(similarities_array))) > 0).nonzero()[0] + 1
        #print([gap_times[min] for min in minimas])
        print ("%s topics were found" % (len(minimas)))
        for min_index in minimas:
            print("%s:%s or %s seconds" % (int(round(gap_times[min_index]/60)),gap_times[min_index]%60,gap_times[min_index]))
    
    
    '''
        compare the shifts of topic from our results compared to the ground base
    '''
    @staticmethod
    def boundryevaluation(curresults,ground_base,accurrcy_shift = 15):
        if len(curresults) != len(ground_base):
            print("the number of the topic in the given results is not the same as the number of topics in the ground base:")
            print("results: %s , ground base: %s" %(len(curresults),len(ground_base)))
        true_positive = 0
        false_negative = 0
        false_positive = 0 
        
        print("The results:")
        print(curresults)
        print("The ground base:")
        print(ground_base)
        
        true_positive_list_debug = []
        #false_negative_list_debug = []
        false_positive_list_debug = []
        
        for grb in ground_base:
            is_false_positive = True
            for res in curresults:
                if abs(res - grb) < accurrcy_shift:
                    is_false_positive = False
                    true_positive_list_debug.append(res)
                    true_positive+=1
                    curresults.remove(res)
                    break   
            if is_false_positive:
                false_positive_list_debug.append(grb)
                false_positive+=1
                
        print("TP: " + str(true_positive_list_debug))
        print("FP: " + str(false_positive_list_debug))
        print("FN: " + str(curresults))
        
        false_negative = len(curresults) # make sure the true positive are removed from here 
        recall_rate = true_positive/(true_positive + false_negative )
        precision_rate = true_positive/(true_positive + false_positive)
        print("TP: %s , FP: %s, FN: %s" %(true_positive,false_positive,false_negative))
        print("precision rate : %s, recall rate : %s " % (precision_rate,recall_rate))
        
        
    '''
        Search for change label after running spectral clustering
    '''
    @staticmethod
    def boundery_change(block_labeles,block_gap_times):
        results = []
        for blk_index in range(len(block_labeles) - 1):
            if block_labeles[blk_index] != block_labeles[blk_index + 1]:
                results.append(int(block_gap_times[blk_index]))
                print("Change label on time %s:%s from %s to %s" % \
                      (int(block_gap_times[blk_index]/60),\
                       int(block_gap_times[blk_index]%60),block_labeles[blk_index],\
                      block_labeles[blk_index + 1]))
        #print(results)
        return results
    @staticmethod
    def save_to_csv():
        pass
    
    
    @staticmethod
    def label_block_timestamp(block_labeles,block_times):
        labels = np.unique(np.array(block_labeles))
        print(len(block_times))
        print(len(block_labeles))
        #start_times = [_time[0] for _time in block_times]
        #end_times = [_time[1] for _time in block_times]
        times = {}
        block_labeles = list(block_labeles)
        for lb in labels:
            first_index = block_labeles.index(lb)
            last_index = len(block_labeles) - 1 - block_labeles[::-1].index(lb)
            start_time = block_times[first_index][0]
            end_time = block_times[last_index][1]
            print("label :%s , starts : %s , ends: %s" %(str(lb),str(start_time),str(end_time)))
            times[lb] = (start_time,end_time)
import seaborn as sns
class MyPlotting():
    @staticmethod
    def similarity_matrix(mydata):
        ax = sns.heatmap(
        mydata, 
        vmin=0, vmax=1, center=0.5,
        cmap=sns.diverging_palette(20, 220, n=200),
        square=True
        )
        ax.set_xticklabels(
            ax.get_xticklabels(),
            rotation=45,
            horizontalalignment='right'
        );
similarity_matrix = []
time_matrix =[]
import pandas as pd<jupyter_output><empty_output><jupyter_text>sliding Window (unforegion blocks),TF,cosine,minimas delimiter<jupyter_code>tt_block = CreateBlocks(transcripts_jsons,window_size=40)
tt_block_TextTiling_blocks = tt_block.partion_by_sliding_windows(120, 20)
'''Calculate Vectors'''
tt_block_TextTiling_tf_calculator = TFCalculator(tt_block_TextTiling_blocks)
tt_block_TextTiling_vectors_matrix = tt_block_TextTiling_tf_calculator.calc()
''' Calculate Similarity'''
tt_block_TextTiling_similarity_handler = DirectSimilarity(tt_block_TextTiling_vectors_matrix)
tt_block_TextTiling_similarities_array = tt_block_TextTiling_similarity_handler.calc()
print(tt_block_TextTiling_similarities_array)
import matplotlib.pyplot as plt
plt.plot(range(len(tt_block_TextTiling_similarities_array)),tt_block_TextTiling_similarities_array,'o-')
plt.show()<jupyter_output><empty_output><jupyter_text>Chunking,TF,Cosine,Minimas delimiter (kind of TextTiling)<jupyter_code>''' Segmenting the Transcript'''
TextTiling_block_handler = CreateBlocks(transcripts_jsons,window_size=30)
TextTiling_blocks = TextTiling_block_handler.partion()
'''Calculate Vectors'''
TextTiling_tf_calculator = TFCalculator(TextTiling_blocks)
TextTiling_vectors_matrix = TextTiling_tf_calculator.calc()
''' Calculate Similarity'''
TextTiling_similarity_handler = DirectSimilarity(TextTiling_vectors_matrix)
TextTiling_similarities_array = TextTiling_similarity_handler.calc()
textTiliing_df = pd.DataFrame(data=TextTiling_similarities_array)
textTiliing_df.describe()
''' For debug and illustration'''
import matplotlib.pyplot as plt
plt.plot(TextTiling_block_handler.get_block_gap_timestamp(),TextTiling_similarities_array,'o-')
plt.show()
'''time_matrix.append(TextTiling_block_handler.get_block_gap_timestamp())
similarity_matrix.append(TextTiling_similarities_array)'''
# find the clusters by minima boundries and then plot the segments upon the time series
TextTilingClustering.find_cluster(TextTiling_similarities_array,TextTiling_block_handler.get_block_gap_timestamp())<jupyter_output>8 topics were found
1:33.114999999999995 or 33.114999999999995 seconds
1:21.810000000000002 or 81.81 seconds
2:3.510000000000005 or 123.51 seconds
3:45.70999999999998 or 165.70999999999998 seconds
3:11.735000000000014 or 191.735 seconds
4:40.81 or 220.81 seconds
5:36.495000000000005 or 276.495 seconds
6:52.13499999999999 or 352.135 seconds<jupyter_text>Chunking,TF-IDF,Cosine,Spectrul Clustering<jupyter_code>''' Segmenting the Transcript'''
sc_block_handler = CreateBlocks(transcripts_jsons,window_size=30) # window_size=50
sc_blocks = sc_block_handler.partion()
'''Calculate matrixes'''
sc_tf_idf_calculator = TFIDFCalculator(sc_blocks)
sc_vectors_matrix = sc_tf_idf_calculator.calc()
sc_tf_calculator = TFCalculator(sc_blocks)
sc_tf_vectors_matrix = sc_tf_calculator.calc()
'''calculate similarity'''
mat_similarity = cosine_similarity(sc_vectors_matrix,sc_vectors_matrix)
tf_mat_similarity = cosine_similarity(sc_tf_vectors_matrix,sc_tf_vectors_matrix)
sc_df = pd.DataFrame(data=mat_similarity)
sc_tf_df = pd.DataFrame(data=tf_mat_similarity)
#sc_df.var()
sc_df.describe()
#sc_tf_df.describe()
MyPlotting.similarity_matrix(mat_similarity)
MyPlotting.similarity_matrix(tf_mat_similarity)
my_thersh = 0.3
thersh_sim_matrix1 = np.asarray([ 0 if sim < my_thersh else sim[0] \
                                for sim in mat_similarity.reshape(-1,1)]).\
reshape(mat_similarity.shape)
MyPlotting.similarity_matrix(thersh_sim_matrix1)
'''Spectral clustering'''
from sklearn.cluster import SpectralClustering
l = 2
sc = SpectralClustering(n_clusters=l).fit(mat_similarity)
sc.labels_<jupyter_output><empty_output><jupyter_text>Chunking,TF-IDF,Cosine,DBscan<jupyter_code>''' Segmenting the Transcript'''
db_block_handler = CreateBlocks(transcripts_jsons,window_size=40)
db_blocks = db_block_handler.partion()
'''Calculate matrixes'''
db_tf_calculator = TFIDFCalculator(db_blocks)
db_vectors_matrix = db_tf_calculator.calc()
'''calculate similarity'''
mat_similarity = cosine_similarity(db_vectors_matrix,db_vectors_matrix)
'''DBscan clustering'''
from sklearn.cluster import DBSCAN
dbscan_clustering = DBSCAN(eps=1.2,min_samples=2).fit(mat_similarity)
dbscan_clustering.labels_<jupyter_output><empty_output><jupyter_text>Word Embedding(Wikipedia),wm distance,<jupyter_code>word_embedding_block_handler = CreateBlocks(transcripts_jsons,window_size=30)
word_embedding_blocks = word_embedding_block_handler.partion()
#word_embedding_block_timestamp = word_embedding_block_handler.get()
import gensim 
from gensim.models import Word2Vec


import gensim.downloader as api

wiki = api.load('glove-wiki-gigaword-300')#,return_path=True<jupyter_output><empty_output><jupyter_text>model = Word2Vec(word_embedding_blocks,min_count=1,sg=1) Calculate Similarity<jupyter_code>#!pip install pyemd
wmd_mat = np.ndarray(shape = (len(word_embedding_blocks),len(word_embedding_blocks)))
for i in range(len(word_embedding_blocks)):
    for j in range(len(word_embedding_blocks)):
        wmd_mat[i][j] = wiki.wmdistance(word_embedding_blocks[i],word_embedding_blocks[j])
pd.DataFrame(data=wmd_mat).describe()
ax = sns.heatmap(
        wmd_mat, 
        vmin=0, vmax=10, center=5,
        cmap=sns.diverging_palette(20, 220, n=200),
        square=True
        )
ax.set_xticklabels(
            ax.get_xticklabels(),
            rotation=45,
            horizontalalignment='right'
);
# try to represent a block using the mean of the vectors
word2vec_mat = [[print(word) for word in block] for block in word_embedding_blocks]<jupyter_output>look
ignition
look
ignition
reverse
process
quench
look
number
different
process
process
sameidea
could
think
parallel
plate
say
yesterdayis
global
energy
balance
balance
could
local
energybalance
let
look
different
approach
get
resultsagain
similar
similar
result
consider
gas
betweentwo
infinite
parallel
plate
say
ignition
criterion
criterion
list
ignition
need
ddxkdtdxwq
thisis
use
different
symbol
reaction
youcould
youcould
say
loss
equal
chemical
reaction
rate
right
equal
assume
go
go
aeert
rightkeep
mind
run
symbol
use
current
samekind
symbol
time
time
say
fine
reason
wehave
write
dot
differential
equation
temperatureand
temperature
temperature
show
mean
integrate
speakingwe
think
bring
take
integratingand
course
involved
involved
process
let
try
see
manipulate
little
bit
introduce
non
length
length
let
suppose
stand
departure
wall
gas
let
say
use
keep
keep
write
note
say
small
compare
mean
look
small
departure
temperature
temperature
exponential
negative
write
sothis
would
expone[...]<jupyter_text>Word embedding(wikipedia), cosine + tf-idf,<jupyter_code>we_block_handler = CreateBlocks(transcripts_jsons,window_size=30)
we_blocks = word_embedding_block_handler.partion()
we_tf_idf_calculator = TFIDFCalculator(word_embedding_blocks)
we_vectors_matrix = we_tf_idf_calculator.calc()
tf_idf_df = pd.DataFrame(we_vectors_matrix,columns= we_tf_idf_calculator.vectorizer.get_feature_names())
tf_idf_df.describe()
we_mat = [] #np.ndarray(len(word_embedding_blocks))
for i in range(len(word_embedding_blocks)):
    block_vector = []
    for j in range(len(word_embedding_blocks[i])):
        try:
            word_vect = wiki[word_embedding_blocks[i][j]]
            word_vect = word_vect * tf_idf_df[word_embedding_blocks[i][j]][i]
            block_vector.append(word_vect)
        except Exception:
            pass
    we_mat.append(sum(block_vector)/len(block_vector))
print(len(we_mat))
#print(we_mat[0])
we_mat_similarity = cosine_similarity(we_mat,we_mat)
MyPlotting.similarity_matrix(we_mat_similarity)<jupyter_output><empty_output><jupyter_text>Sliding window,TF, cosine, Spectral clustering<jupyter_code>sw_block_handler = CreateBlocks(transcripts_jsons,window_size=30) # window_size=50
sw_blocks = sw_block_handler.partion_by_sliding_windows(40,20)
print(len(sw_blocks))
print(len(sw_block_handler.get_block_timestamp()))
'''Calculate matrixes'''
sw_tf_idf_calculator = TFIDFCalculator(list(sw_blocks))
sw_vectors_matrix = sw_tf_idf_calculator.calc()
sw_tf_calculator = TFCalculator(sw_blocks)
sw_tf_vectors_matrix = sw_tf_calculator.calc()
'''calculate similarity'''
sw_mat_similarity = cosine_similarity(sw_vectors_matrix,sw_vectors_matrix)
sw_tf_mat_similarity = cosine_similarity(sw_tf_vectors_matrix,sw_tf_vectors_matrix)
sw_df = pd.DataFrame(data=sw_mat_similarity)
sw_tf_df = pd.DataFrame(data=sw_tf_mat_similarity)
MyPlotting.similarity_matrix(sw_mat_similarity)
MyPlotting.similarity_matrix(sw_tf_mat_similarity)
my_thersh = 0.4
thersh_sim_matrix = np.asarray([ 0 if sim < my_thersh else sim[0] \
                                for sim in sw_mat_similarity.reshape(-1,1)]).\
reshape(sw_mat_similarity.shape)
MyPlotting.similarity_matrix(thersh_sim_matrix)
l = 14
sc = SpectralClustering(n_clusters=l).fit(thersh_sim_matrix)
print(sc.labels_)
print(len(sc.labels_))
my_eval = TextTilingClustering.boundery_change(sc.labels_,sw_block_handler.get_block_gap_timestamp())
len(my_eval)
TextTilingClustering.boundryevaluation(my_eval,ground_base_kernel_autogenerate)<jupyter_output>The results:
[96, 174, 275, 351, 424, 474, 533, 615, 670, 716, 775, 888, 934, 1039]
The ground base:
[22, 87, 207, 270, 319, 370, 450, 558, 602, 671, 756, 930, 978, 1011]
TP: [96, 275, 615, 670, 934]
FP: [22, 207, 319, 370, 450, 558, 756, 978, 1011]
FN: [174, 351, 424, 474, 533, 716, 775, 888, 1039]
TP: 5 , FP: 9, FN: 9
precision rate : 0.35714285714285715, recall rate : 0.35714285714285715<jupyter_text>Sliding window,word2vec,wmdistance<jupyter_code>w2v_block_handler = CreateBlocks(transcripts_jsons,window_size=30)
w2v_blocks = w2v_block_handler.partion()
wmd_mat = np.ndarray(shape = (len(w2v_blocks),len(w2v_blocks)))
for i in range(len(w2v_blocks)):
    for j in range(len(w2v_blocks)):
        wmd_mat[i][j] = wiki.wmdistance(w2v_blocks[i],w2v_blocks[j])
my_thersh = 5
thersh_wmd_matrix = np.asarray([ 10 if sim > my_thersh else sim[0] \
                                for sim in wmd_mat.reshape(-1,1)]).\
reshape(wmd_mat.shape)
ax = sns.heatmap(
        wmd_mat, 
        vmin=0, vmax=10, center=5,
        cmap=sns.diverging_palette(20, 220, n=200),
        square=True
        )
ax.set_xticklabels(
            ax.get_xticklabels(),
            rotation=45,
            horizontalalignment='right'
);
ax = sns.heatmap(
        thersh_wmd_matrix, 
        vmin=0, vmax=10, center=5,
        cmap=sns.diverging_palette(20, 220, n=200),
        square=True
        )
ax.set_xticklabels(
            ax.get_xticklabels(),
            rotation=45,
            horizontalalignment='right'
);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09502948609941028, 'fraction_numerical': 0.03201347935973041, 'mean_word_length': 4.20035046728972, 'pattern_counts': {'":': 0, '<': 41, '<?xml version=': 0, '>': 40, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 9, 'word_list_counts': {'cursed_substrings.json': 9, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '10', 'dir': 'jupyter-structured-clean-dedup', 'id': '432200', 'license': 'MIT', 'n_tokens_mistral': 6619, 'n_tokens_neox': 5924, 'n_words': 1142, 'path': 'poc/Framework.ipynb', 'repo_name': 'yanivohayon8/CLMN7-DSC-PROJECT', 'size': '17805'}