<jupyter_start><jupyter_text><jupyter_code>import pandas as pd 
import numpy as np
import os


import statsmodels
import statsmodels.api as sm
import scipy.stats as stats

import matplotlib.pyplot as plt

import re
from collections import Counter
from scipy import sparse
import networkx as nx
from google.colab import drive
drive.mount('/content/gdrive')
# import the csv file with comment and display the pandas dataframe
comm = pd.read_csv(r"/content/gdrive/MyDrive/com_liwc.csv", sep='\t', engine='python')
df = pd.DataFrame(data=comm)

pd.set_option('display.max_columns', None)
df
#delete the NaN values in dataframe
print('df shape before dropping row: \t', df.shape)
df = df[df['c_text'].notna()]
print('df shape after dropping row: \t', df.shape)
print('Number of Nan in comments text: ', df['c_text'].isna().sum())
df.shape
#dataframe extraction by political side
'''dx_sx = df['p_dx_sx'].unique().tolist()
print(dx_sx)
df_0 = df[df['p_dx_sx'] == 'destra']
df_1 = df[df['p_dx_sx'] == 'sinistra']'''

#dataframe extraction by campaign
campaign = df['p_camapagna2'].unique().tolist()
print(campaign)
df_0 = df[df['p_camapagna2'] == 'Neg-comp']
df_1 = df[df['p_camapagna2'] == 'Positiva']
#dataframe whit only comments df_0
comments_0 = df_0[['c_text']].copy()
comments_0.rename(columns={'c_text':'text'}, inplace=True)

#dataframe whit only comments df_1
comments_1 = df_1[['c_text']].copy()
comments_1.rename(columns={'c_text':'text'}, inplace=True)<jupyter_output><empty_output><jupyter_text>Hashtag extraction and dataframe creation<jupyter_code>#return counter of pattern appearance in a text 
def pattern_freq_comment(text, pattern):
    p_freq = Counter()
    p_found= re.findall(pattern, text)
    for p in p_found:
        p_freq[p.lower()] += 1
    return p_freq  

#return counter of pattern appearance in a array of text 
def pattern_freq(docs, pattern):
    p_freq = Counter()
    for text in docs:
        p_found= re.findall(pattern, text)
        for p in p_found:
            p_freq[p.lower()] += 1
    return p_freq 

#select only topic that appears at leat "threshold" times
def hash_extraction(comments,treshold):
  hashtags_patt = re.compile(r'#[\w]*')
  hashtags_freq = pattern_freq(comments['text'].values, hashtags_patt) #hashtag counter with all the hashtags 
  #threshold=2
  hash_dict = {key:val for key, val in hashtags_freq.items() if val > treshold} #we extract the hashtags that appear at minimum threshold times
  return hash_dict


def database_creation(df, hash_dict):
  mask = df.c_text.str.contains('#[\w]*') 
  df_hash = df[mask]
  df_hash = df_hash.reset_index()
  df_hash = df_hash.drop('index',axis = 1)      
  #select subdataframe where at least 1 hastag is contained in the hash_dict (hashtags that appear more than threshold time)
  df_hash_final= df_hash.copy()
  indexes= []
  hashtags_patt = re.compile(r'#[\w]*')
  for index, row in df_hash_final.iterrows():
    present = False
    hash_in_com=pattern_freq_comment(df_hash_final['c_text'][index],hashtags_patt)
    for key in hash_in_com:
        if key in hash_dict.keys():
            present=True
    if not present:
        #print("drop")
        indexes.append(index)

  for i in range(len(indexes)):  
    #df_hash_final = df_hash_final.drop(df_hash_final.index[indexes[i]])
    df_hash_final = df_hash_final.drop([indexes[i]])
  df_hash_final = df_hash_final.reset_index()
  df_hash_final = df_hash_final.drop('index',axis = 1)

  return df_hash_final<jupyter_output><empty_output><jupyter_text>Graph creation<jupyter_code>#biadjacency graph creation
def biadjacency(df,feature):
    emo_dict = dict(zip(feature,[0 for x in range(0,len(feature))]))
    size_df= len(df)
    matrix = np.zeros((len(feature),size_df)) 
    hashtags_patt = re.compile(r'#[\w]*')
    hash_list = list(feature.keys())
    for i in range(size_df):
    
        comm = df['c_text'][i]
        level = df['c_Emo_Neg'][i]
        hashtags_freq_in_com = pattern_freq_comment(comm, hashtags_patt)
        hash = list(hashtags_freq_in_com.keys())
        #print(comm)
        #print(hash)
        for j in range(len(hash)):
            temp = hash[j]
            if hash_list.count(temp) > 0:
              if temp in emo_dict:
                temp_value = emo_dict[temp]
                emo_dict[temp] = temp_value+level
              index = hash_list.index(temp)
              matrix[index][i]+=1 
    matrix_sparse = sparse.csr_matrix(matrix)
    #print(matrix_sparse)
    biadjacency_graph = nx.bipartite.from_biadjacency_matrix(matrix_sparse)
    return biadjacency_graph,emo_dict


#emo_neg normalized dictionary
def emo_neg_dict(emo_dict):
  inverse = [(value, key) for key, value in emotion_dict.items()]
  emo_final = {k: v / total for total in (max(emotion_dict.values()),) for k, v in emotion_dict.items()}
  return emo_final


#bipartite nodes and edges creation
def bipartite_att(df,feature,biad_graph, emo_final): #feature is the hashtag_list
  set1 = pd.DataFrame(feature.keys(),columns =['Label'])
  set1['type']="hashtag"  
  set1['Emo_Neg']= emo_final.values()
  #l = [x for x in range(len(feature1))]
  #set1['Id'] = l

  set2 = df[['c_topic',"p_PARTITO", 'p_politician','c_Emo_Neg','c_Rabbia']]  #change or add attribute attribute 
  set2.reset_index(inplace=True)
  set2['type'] = 'comment'  
  set2.rename(columns={ 'p_PARTITO': 'partito', 'p_politician': 'politician','c_Emo_Neg':'Emo_Neg','c_Rabbia':'Rabbia'}, inplace=True) #change attribute names
  print(len(set1))
  print(len(set2))
  nodes = set1.append(set2)

  l = [x for x in range(len(df)+len(feature))]
  nodes['Id'] = l

  edge = nx.to_pandas_edgelist(biad_graph)

  return nodes,edge


#projection with respect to #
def proj(feature,biad_graph):
  nodes = []
  for i in range(len(feature)):
    nodes.append(i)
  projection = nx.bipartite.projection.weighted_projected_graph(biad_graph, nodes)
  return projection

#projection nodes and edges creation
def proj_att(feature,projection,emo_final): #feature should be the hashtags dictionnary here
  nodes = pd.DataFrame({'Label': feature})
  l = [x for x in range(len(nodes))]
  nodes['Id'] = l
  nodes['Label'] = feature
  
  nodes['Emo_Neg'] = emo_final.values() #added
  edge = nx.to_pandas_edgelist(projection)
  return nodes,edge
treshold = 2
hash_dict_0 = hash_extraction(comments_0,treshold)
df_0_final = database_creation(df_0, hash_dict_0)

hash_dict_1 = hash_extraction(comments_1,treshold)
df_1_final = database_creation(df_1, hash_dict_1)
biad_graph, emotion_dict = biadjacency(df_0_final,hash_dict_0)
emo_final = emo_neg_dict(emotion_dict)
nodes, edges = bipartite_att(df_0_final,hash_dict_0,biad_graph,emo_final)

nodes.to_csv("nodelist_bipartite_hash_set0.csv",index=False)
edges.to_csv("edgelist_bipartite_hash_set0.csv",index=False)
biad_graph, emotion_dict = biadjacency(df_0_final,hash_dict_0)
emo_final = emo_neg_dict(emotion_dict)
proj_graph = proj(hash_dict_0,biad_graph)
nodes, edges = proj_att(hash_dict_0,proj_graph,emo_final)

nodes.to_csv("nodelist_projection_hash_set0.csv",index=False)
edges.to_csv("edgelist_projection_hash_set0.csv",index=False)
biad_graph, emotion_dict = biadjacency(df_1_final,hash_dict_1)
emo_final = emo_neg_dict(emotion_dict)
nodes, edges = bipartite_att(df_1_final,hash_dict_1,biad_graph,emo_final)

nodes.to_csv("nodelist_bipartite_hash_set1.csv",index=False)
edges.to_csv("edgelist_bipartite_hash_set1.csv",index=False)
biad_graph, emotion_dict = biadjacency(df_1_final,hash_dict_1)
emo_final = emo_neg_dict(emotion_dict)
proj_graph = proj(hash_dict_1,biad_graph)
nodes, edges = proj_att(hash_dict_1,proj_graph,emo_final)

nodes.to_csv("nodelist_projection_hash_set1.csv",index=False)
edges.to_csv("edgelist_projection_hash_set1.csv",index=False)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12347423872542722, 'fraction_numerical': 0.009379416677373763, 'mean_word_length': 5.086004691164972, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '325847', 'license': 'CC0-1.0', 'n_tokens_mistral': 2947, 'n_tokens_neox': 2865, 'n_words': 571, 'path': 'Post_Group/Hashtag_network.ipynb', 'repo_name': 'Lally09-fx/net2020', 'size': '7783'}