<jupyter_start><jupyter_text>Session 5: Network Analysis<jupyter_code># import dependencies 
import os

# data anaylsis 
import pandas as pd
from tqdm import tqdm # feedback (progression bar)

# NLP
import spacy
nlp = spacy.load("en_core_web_sm")<jupyter_output><empty_output><jupyter_text>Read data<jupyter_code>input_file = os.path.join("..", "data", "fake_or_real_news.csv")
data = pd.read_csv(input_file)<jupyter_output><empty_output><jupyter_text>__Filter dataframe__<jupyter_code>real_df = data[data["label"]=="REAL"]["text"]
real_df.sample(2)<jupyter_output><empty_output><jupyter_text>Extract entities<jupyter_code>post_entities = []

for post in tqdm(real_df):
    # create temporary list
    tmp_list = []
    # create spacy doc object
    doc = nlp(post)
    # for every named entitiy in the doc
    for entitiy in doc.ents:
        if entitiy.label_ == "PERSON":
            tmp_list.append(entitiy.text)
    # add tmp_list to main list
    post_entities.append(tmp_list)<jupyter_output>100%|██████████| 3171/3171 [08:08<00:00,  6.49it/s]<jupyter_text>Create edgelist using `itertools.combinations()`<jupyter_code>from itertools import combinations
edgelist = []

# iterate over every document 
for doc in post_entities:
    # take all combinations in the doc in pairs
    edges = list(combinations(doc, 2))
    # for each combination (pair)
    for edge in edges:
        # append this to final edge list
        edgelist.append(tuple(sorted(edge)))
        # sorted sorts them so that we don't have duplicates in different order
print(sorted(("Jane", "Mary")))
print(sorted(("Mary", "Jane")))<jupyter_output>['Jane', 'Mary']
['Jane', 'Mary']<jupyter_text>Counting edges<jupyter_code>from collections import Counter
# count all edges, and return the 10 most common
Counter(edgelist).most_common(10)
counted_edges = []

for pair, weight in Counter(edgelist).items():
    nodeA = pair[0]
    nodeB = pair[1]
    counted_edges.append((nodeA, nodeB, weight))
    
# weight = total occurance they appear togehter across the data
len(counted_edges)<jupyter_output><empty_output><jupyter_text>Create dataframe<jupyter_code>edges_df = pd.DataFrame(counted_edges, columns=["nodeA", "nodeB", "weight"])
filtered_df = edges_df[edges_df["weight"] > 500]
# save dataframe for assignment
edges_df.to_csv("../data/weighted_edgelist_realnews.csv")<jupyter_output><empty_output><jupyter_text>Create network<jupyter_code># import saved file 
df = pd.read_csv("../data/weighted_edgelist_realnews.csv")
filtered_df = df[df["weight"] > 500]
import networkx as nx
import matplotlib.pyplot as plt
import math
G = nx.from_pandas_edgelist(filtered_df, "nodeA", "nodeB", ["weight"])
# plot it
pos = nx.nx_agraph.graphviz_layout(G, prog = "neato")
# draw
nx.draw(G, pos, with_labels=True, node_size=20, font_size=10)
# save 
#plt.savefig("network.png", dpi=300, bbox_inches="tight")
plt.figure(figsize=(18,18))
spring_layout = nx.spring_layout(G, k=math.sqrt(G.order()))
nx.draw_networkx_labels(G, spring_layout, font_size=13, verticalalignment="bottom", font_weight="semibold")
nx.draw_networkx_edges(G, spring_layout, alpha = 0.5)
nx.draw_networkx_nodes(G, spring_layout, node_size=120, node_color="steelblue")
plt.show()<jupyter_output><empty_output><jupyter_text>Centrality measures<jupyter_code>bc_metric = nx.betweenness_centrality(G)
ev_metric = nx.eigenvector_centrality(G)
bc_metric
pd.DataFrame(bc_metric.items(), columns=["node", "betweennnes"]).sort_values("betweennnes", ascending=False)
pd.DataFrame(ev_metric.items(), columns=["node", "eigenvector"]).sort_values("eigenvector", ascending=False)
pd.read_csv("../assignments/4-network/output/centrality_measures.csv")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1350108225108225, 'fraction_numerical': 0.015151515151515152, 'mean_word_length': 6.514227642276423, 'pattern_counts': {'":': 1, '<': 36, '<?xml version=': 0, '>': 37, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '574011', 'license': 'MIT', 'n_tokens_mistral': 1405, 'n_tokens_neox': 1336, 'n_words': 256, 'path': 'notebooks/session6_nd.ipynb', 'repo_name': 'NicoleDwe/cds-language', 'size': '3696'}