<jupyter_start><jupyter_text>Web and Social Project Codeusing Python 2.7<jupyter_code>%matplotlib inline
import pandas as pd
from collections import Counter
import matplotlib.pyplot as plt
import numpy as np
import networkx as nx
data  = open("Data/msnbc990928.seq")
lines = data.readlines()
lines[:10]<jupyter_output><empty_output><jupyter_text>Data<jupyter_code>data2 = [lin.strip().split() for lin in lines]
data2[:5]    
len_line=[len(lin) for lin in data2]
print len_line[:5]
print sum(len_line)/len(len_line)
pd.DataFrame(len_line).describe().to_csv("Orig_metrics.csv")<jupyter_output><empty_output><jupyter_text>Most Common Landing Page<jupyter_code>first_pages  =  [lin[0] for lin in data2]
first_pages[:10]
Counter(first_pages).most_common()
first_pages_dict = Counter(first_pages)<jupyter_output><empty_output><jupyter_text>Scatter plot to show frequency of page when it is a landing page<jupyter_code>plt.scatter(x = list(first_pages_dict.keys()), y=list(first_pages_dict.values()))
plt.show()<jupyter_output><empty_output><jupyter_text>Scatter plot to show frequency of page when it is an exit page<jupyter_code>last_pages  =  [lin[-1] for lin in data2]
last_pages[:10]
Counter(last_pages).most_common()
last_pages_dict = Counter(last_pages)
plt.scatter(x = list(last_pages_dict.keys()), y=list(last_pages_dict.values()))
plt.show()
da = np.array(data2)<jupyter_output><empty_output><jupyter_text>Max and Min No of Page Views<jupyter_code>cnt = Counter(np.concatenate(da))
cnt.most_common()
sorted(Counter(np.concatenate(da)).keys())
np.concatenate(da).shape
pages = ['1','2', '3', '4', '5', '6','7', '8', '9', '10', 
         '11', '12', '13', '14', '15', '16', '17']<jupyter_output><empty_output><jupyter_text>Exit Rate<jupyter_code>exit_rate={}
for page in pages:
    # Number of times a page appears in the sequence
    page_true = len([lin for lin in data2 if page in lin ])
    # Number of times a page is an exit page
    exit_pages_no = len([lin for lin in data2 if page == lin[-1] ])
    exit_rate[page] = exit_pages_no / (page_true*1.0)
exit_rate
tab = pd.DataFrame()
tab['pages'] = sorted(exit_rate.keys())
tab
tab['exit_rate']=[exit_rate[i] for i in sorted(exit_rate.keys())]
tab.T<jupyter_output><empty_output><jupyter_text>Bounce Rate<jupyter_code>bounce_rate={}
for page in pages:
    # Number of times a page is bounced
    bounce_pages_no = len([lin for lin in data2 if len(lin)==1 and lin[0] == page])
    # Number of times a page is a potential bounce
    page_b_true = len([lin for lin in data2 if lin[0]==page])
    bounce_rate[page] = bounce_pages_no / (page_b_true*1.0)
bounce_rate
tab['bounce_rate']=[bounce_rate[i] for i in sorted(bounce_rate.keys())]
tab['page_counts']=[cnt[i] for i in sorted(cnt.keys())]
tab.set_index('pages').T.describe()
tab.to_csv("metrics.csv")
tab.set_index('pages').T.describe().to_csv("metrics_desc.csv")
tab.set_index('pages').T
tab.T<jupyter_output><empty_output><jupyter_text>Incidence Matrix Code<jupyter_code>incidence_matrix= np.zeros((len(pages), len(pages)))<jupyter_output><empty_output><jupyter_text>Pages Name<jupyter_code>pages_name = "frontpage news tech local opinion on-air misc weather msn-news health living business msn-sports sports summary bbs travel".split()
pages_name, len(pages_name)<jupyter_output><empty_output><jupyter_text>Mapping Pages and Page Names<jupyter_code>mapping = dict(zip(pages, range(len(pages))))
mapping
 map_df = pd.DataFrame([i for i in sorted(mapping.values())])
mapping_original = dict((zip(range(len(pages_name)), pages_name)))
mapping_original
map_df['pages']=[mapping_original[i] for i in sorted(mapping_original.keys())]
map_df.to_csv("mappings.csv")<jupyter_output><empty_output><jupyter_text>Create an incidence matrix based on co-occurrence<jupyter_code>incidence_matrix= np.zeros((len(pages), len(pages)))

for line in data2:  # For Each seq in the file
    length = len(line)
    if length>=2:
        #print line
        for i in range(length-1):  # for each element in the file 
            tmp = (line[i], line[i+1])
            incidence_matrix[mapping[tmp[0]], mapping[tmp[1]]]+=1
                        #print k, j , tmp

print(incidence_matrix.shape)
incidence_matrix.mean(), incidence_matrix.min(), incidence_matrix.max()
incidence_matrix.shape
matrix2 = incidence_matrix >= incidence_matrix.mean() 
matrix2.shape
G = nx.from_numpy_matrix(incidence_matrix)
nx.info(G)
G = nx.relabel_nodes(G, mapping_original)
type(G)
G.is_directed()
G.degree()
nx.degree_centrality(G)
nx.betweenness_centrality(G)
nx.eigenvector_centrality(G)
G.edges(data=True)[:10]
nx.connectivity.average_node_connectivity(G)
nx.closeness_centrality(G)
list(nx.clique.find_cliques(G))
nx.pagerank(G, alpha=1)<jupyter_output><empty_output><jupyter_text>Graph based on Incidence matrix with threshod<jupyter_code>thres = incidence_matrix.mean()
thres
G = nx.from_numpy_matrix(incidence_matrix>=thres)
nx.info(G)
G = nx.relabel_nodes(G, mapping_original)
type(G)
G.is_directed()
deg = G.degree()
dc = nx.degree_centrality(G)
bc = nx.betweenness_centrality(G)
ec=nx.eigenvector_centrality(G)
cc=nx.closeness_centrality(G)
G.edges(data=True)[:10]
list(nx.clique.find_cliques(G))
pr = nx.pagerank(G, alpha=1, max_iter=1000)
nx.density(G)
tabc = pd.DataFrame()
tabc['pages'] = sorted(G.nodes())
tabc = tabc.set_index('pages')
tabc.T
tabc['degree_centrality']= [dc[i] for i in sorted(dc.keys())]
tabc
tabc['degree']= [deg[i] for i in sorted(deg.keys())]
tabc
tabc['betweeness_centrality']= [bc[i] for i in sorted(bc.keys())]
tabc
tabc['eigenVector_centrality']= [ec[i] for i in sorted(ec.keys())]
tabc
tabc['closeness_centrality']= [cc[i] for i in sorted(cc.keys())]
tabc
tabc['page_rank']= [pr[i] for i in sorted(pr.keys())]
tabc
tabc.to_csv("Graph_metrics.csv")
tabc.describe().to_csv("Graph_metrics_desc.csv")
tabc.T
G.nodes()<jupyter_output><empty_output><jupyter_text>For Finding the suggestion Triads<jupyter_code>def get_recommendation(pair, data):
    '''Find the most common triad for the given pair of pages, return as a list of values'''
    most_occured_page=[]
    occured_page=[]
    for line in data:
        if len(line)>=3:
            #print line
            length= len(line)
            for i in range(length):
                if i < length-2:
                    tmp = (line[i], line[i+1], line[i+2])
                    #print 'tmp', tmp, pair, set(tmp).intersection(set(pair))
                    if pair[0]==tmp[0] and pair[1]==tmp[1]:
                        #print pair, '-', tmp
                        most_occured_page.append(tmp[-1])
                        occured_page.append(tmp)    
    rec = Counter(occured_page)
    return rec.most_common()
pair =('6', '5')
get_recommendation(pair, data2)
pair = ("2", "5")
get_recommendation(pair, data2)
pd.DataFrame(get_recommendation(("2", "5"), data2)).to_csv("Recommend.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14634855979051498, 'fraction_numerical': 0.014256619144602852, 'mean_word_length': 5.425233644859813, 'pattern_counts': {'":': 0, '<': 58, '<?xml version=': 0, '>': 61, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '14', 'dir': 'jupyter-structured-clean-dedup', 'id': '74621', 'license': 'MIT', 'n_tokens_mistral': 2611, 'n_tokens_neox': 2541, 'n_words': 523, 'path': 'J-PAAC_Code_Notebook.ipynb', 'repo_name': 'abishekganesh72/web-social-analysis-project', 'size': '6874'}