<jupyter_start><jupyter_text>8/22/2020 SummaryAll related code/figures for oSBM for ICWSM revision. Code organization assumes working directory is notebook folder location. Note, this notebook pulls ReFeX generated features which happen in another script which we modify to create different recursions.<jupyter_code>import sys
print(sys.version)
from __future__ import division
import os
#import igraph
import numpy as np
import pandas as pd
from matplotlib.backends.backend_pdf import PdfPages
from scipy.stats.kde import gaussian_kde
from matplotlib.ticker import FixedLocator,LinearLocator,MultipleLocator, FormatStrFormatter
# relevant python libraries
execfile('./functions/python_libraries.py')
# oSBM code
from sklearn.model_selection import RandomizedSearchCV


execfile('./functions/create_oSBM_class_specific_alternative.py')
execfile('./functions/LINK_revised.py')
execfile('./functions/compute_homophily.py')<jupyter_output>2.7.15 |Anaconda, Inc.| (default, Oct 23 2018, 13:35:16) 
[GCC 4.2.1 Compatible Clang 4.0.1 (tags/RELEASE_401/final)]<jupyter_text>create 4 graph instances: noB-noO, yesB-noO, noB-yesO, yesB-yesO  1) Amherst-analog: homophily, equal classes (no ReFeX)<jupyter_code>## note have 500/1500 set-up for comparison as we change homophily
( adj_matrix_gender, gender_y) = create_affiliation_model_temp(84,
                                          lambda_block_parameter =1.1,
                                          dispersion_parameter_vect = np.array([np.array([0, 0]),
                                            np.array([0, 0])]),
                                          class_size_vect = np.array([1000,1000]))
tag = 'Amherst_analog_SBM_by_degree_within_nw'
## WITHIN-NETWORK
gender_y = np.array(map(np.int,gender_y)) ## need np.int for machine precisions reasons
d = {'keys':range(len(gender_y)),
    'gender_y':gender_y}
y_df = pd.DataFrame(d)
Gnx = nx.from_numpy_matrix(adj_matrix_gender,
                           create_using=nx.DiGraph())
fb100_df_tmp = []
for j in nx.generate_edgelist(Gnx, data = False):
    fb100_df_tmp.append(map(np.int,j.split(' ')))
fb100_df = pd.DataFrame(fb100_df_tmp)
fb100_df.columns = ['from','to']
fb100_df_undirect = fb100_df.copy()
fb100_df_undirect['to'] = fb100_df['from']
fb100_df_undirect['from'] = fb100_df['to']
fb100_df = fb100_df.append(fb100_df_undirect)
fb100_df['weight'] = 1
fb100_df.head()
fb100_df.drop_duplicates(inplace = True)
## save output
fb100_df.to_csv('../../data/SBM_ICWSM_revise/train_'+tag+'.csv', header = False, index = False)
y_df.to_csv('../../data/SBM_ICWSM_revise/train_'+tag+'_gender.csv', header = True, index = False)

## Increasing Homophily/Block Structure
## Across-Layer Set-up
for iter_num in range(10):
    print('instance: '+str(iter_num))
    for block in np.array([1.1, 1.2, 1.3, 1.4, 1.5]):
        ( adj_matrix_gender, gender_y) = create_affiliation_model_temp(84,
                                              lambda_block_parameter =block,
                                              dispersion_parameter_vect = np.array([np.array([0, 0]),
                                                np.array([0, 0])]),
                                        class_size_vect = np.array([1000,1000]))

        #print('avg degree',np.mean(np.sum(adj_matrix_gender,1)))
        #print('avg degree among Class 1',np.mean(np.sum(adj_matrix_gender[gender_y==1],1)))
        #print('max degree among Class 1',np.max(np.array(np.sum(adj_matrix_gender[gender_y==1],1).T[0])[0]))

        #print('avg degree among Class 0',np.mean(np.sum(adj_matrix_gender[gender_y==0],1)))
        #print('max degree among Class 0',np.max(np.array(np.sum(adj_matrix_gender[gender_y==0],1).T[0])[0]))

        #print('homophily', homophily_index_Jackson_alternative(adj_matrix_gender, gender_y))
        tag = str(iter_num) + '_Amherst_analog_SBM_across_layer_' + str(block)

        gender_y = np.array(map(np.int,gender_y)) ## need np.int for machine precisions reasons
        d = {'keys':range(len(gender_y)),
            'gender_y':gender_y}
        y_df = pd.DataFrame(d)
        Gnx = nx.from_numpy_matrix(adj_matrix_gender,
                                   create_using=nx.DiGraph())
        fb100_df_tmp = []
        for j in nx.generate_edgelist(Gnx, data = False):
            fb100_df_tmp.append(map(np.int,j.split(' ')))
        fb100_df = pd.DataFrame(fb100_df_tmp)
        fb100_df.columns = ['from','to']
        fb100_df_undirect = fb100_df.copy()
        fb100_df_undirect['to'] = fb100_df['from']
        fb100_df_undirect['from'] = fb100_df['to']
        fb100_df = fb100_df.append(fb100_df_undirect)
        fb100_df['weight'] = 1
        fb100_df.head()
        fb100_df.drop_duplicates(inplace = True)
        fb100_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'.csv', header = False, index = False)
        y_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'_gender.csv', header = True, index = False)

## Increasing Homophily/Block Structure
## Across-Network Set-up
for iter_num in range(10):
    print('instance: '+str(iter_num))
    for block in np.array([1.1, 1.2, 1.3, 1.4, 1.5]):
        ( adj_matrix_gender, gender_y) = create_affiliation_model_temp(84,
                                              lambda_block_parameter =block,
                                              dispersion_parameter_vect = np.array([np.array([0, 0]),
                                                np.array([0, 0])]),
                                        class_size_vect = np.array([500,1500]))

        #print('avg degree',np.mean(np.sum(adj_matrix_gender,1)))
        #print('avg degree among Class 1',np.mean(np.sum(adj_matrix_gender[gender_y==1],1)))
        #print('max degree among Class 1',np.max(np.array(np.sum(adj_matrix_gender[gender_y==1],1).T[0])[0]))

        #print('avg degree among Class 0',np.mean(np.sum(adj_matrix_gender[gender_y==0],1)))
        #print('max degree among Class 0',np.max(np.array(np.sum(adj_matrix_gender[gender_y==0],1).T[0])[0]))

        #print('homophily', homophily_index_Jackson_alternative(adj_matrix_gender, gender_y))
        tag = str(iter_num) + '_Amherst_analog_SBM_across_network_' + str(block)

        gender_y = np.array(map(np.int,gender_y)) ## need np.int for machine precisions reasons
        d = {'keys':range(len(gender_y)),
            'gender_y':gender_y}
        y_df = pd.DataFrame(d)
        Gnx = nx.from_numpy_matrix(adj_matrix_gender,
                                   create_using=nx.DiGraph())
        fb100_df_tmp = []
        for j in nx.generate_edgelist(Gnx, data = False):
            fb100_df_tmp.append(map(np.int,j.split(' ')))
        fb100_df = pd.DataFrame(fb100_df_tmp)
        fb100_df.columns = ['from','to']
        fb100_df_undirect = fb100_df.copy()
        fb100_df_undirect['to'] = fb100_df['from']
        fb100_df_undirect['from'] = fb100_df['to']
        fb100_df = fb100_df.append(fb100_df_undirect)
        fb100_df['weight'] = 1
        fb100_df.head()
        fb100_df.drop_duplicates(inplace = True)
        fb100_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'.csv', header = False, index = False)
        y_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'_gender.csv', header = True, index = False)<jupyter_output>instance: 0
('p_in: ', 0.046200000000000005)
('p_out: ', 0.03499999999999999)

('p_in: ', 0.0504)
('p_out: ', 0.028)

('p_in: ', 0.0546)
('p_out: ', 0.021)

('p_in: ', 0.0588)
('p_out: ', 0.014)

('p_in: ', 0.063)
('p_out: ', 0.007)

instance: 1
('p_in: ', 0.046200000000000005)
('p_out: ', 0.03499999999999999)

('p_in: ', 0.0504)
('p_out: ', 0.028)

('p_in: ', 0.0546)
('p_out: ', 0.021)

('p_in: ', 0.0588)
('p_out: ', 0.014)

('p_in: ', 0.063)
('p_out: ', 0.007)

instance: 2
('p_in: ', 0.046200000000000005)
('p_out: ', 0.03499999999999999)

('p_in: ', 0.0504)
('p_out: ', 0.028)

('p_in: ', 0.0546)
('p_out: ', 0.021)

('p_in: ', 0.0588)
('p_out: ', 0.014)

('p_in: ', 0.063)
('p_out: ', 0.007)

instance: 3
('p_in: ', 0.046200000000000005)
('p_out: ', 0.03499999999999999)

('p_in: ', 0.0504)
('p_out: ', 0.028)

('p_in: ', 0.0546)
('p_out: ', 0.021)

('p_in: ', 0.0588)
('p_out: ', 0.014)

('p_in: ', 0.063)
('p_out: ', 0.007)

instance: 4
('p_in: ', 0.046200000000000005)
('p_out: ', 0.03499[...]<jupyter_text>2) Sloan-analog: slight homophily, un-equal classes (ReFeX)<jupyter_code>(adj_matrix_gender, gender_y) = create_affiliation_model_temp(84,
                                          lambda_block_parameter =1.1,
                                          dispersion_parameter_vect = np.array([np.array([0, 0]),
                                            np.array([0, 0])]),
                                          class_size_vect = np.array([500,1500]))


tag = 'Sloan_analog_SBM_by_degree_within_nw'

## WITHIN-NETWORK
gender_y = np.array(map(np.int,gender_y)) ## need np.int for machine precisions reasons
d = {'keys':range(len(gender_y)),
    'gender_y':gender_y}
y_df = pd.DataFrame(d)
Gnx = nx.from_numpy_matrix(adj_matrix_gender,
                           create_using=nx.DiGraph())
fb100_df_tmp = []
for j in nx.generate_edgelist(Gnx, data = False):
    fb100_df_tmp.append(map(np.int,j.split(' ')))
fb100_df = pd.DataFrame(fb100_df_tmp)
fb100_df.columns = ['from','to']
fb100_df_undirect = fb100_df.copy()
fb100_df_undirect['to'] = fb100_df['from']
fb100_df_undirect['from'] = fb100_df['to']
fb100_df = fb100_df.append(fb100_df_undirect)
fb100_df['weight'] = 1
fb100_df.head()
fb100_df.drop_duplicates(inplace = True)
## save output
fb100_df.to_csv('../../data/SBM_ICWSM_revise/train_'+tag+'.csv', header = False, index = False)
y_df.to_csv('../../data/SBM_ICWSM_revise/train_'+tag+'_gender.csv', header = True, index = False)

## Increasing Homophily/Block Structure
## Across-Layer Set-up
for iter_num in range(10):
    print('instance: '+str(iter_num))
    for block in np.array([1.1, 1.2, 1.3, 1.4, 1.5]):
        ( adj_matrix_gender, gender_y) = create_affiliation_model_temp(84,
                                              lambda_block_parameter =block,
                                              dispersion_parameter_vect = np.array([np.array([0, 0]),
                                                np.array([0, 0])]),
                                        class_size_vect = np.array([500,1500]))

        tag = str(iter_num) + '_Sloan_analog_SBM_across_layer_' + str(block)

        gender_y = np.array(map(np.int,gender_y)) ## need np.int for machine precisions reasons
        d = {'keys':range(len(gender_y)),
            'gender_y':gender_y}
        y_df = pd.DataFrame(d)
        Gnx = nx.from_numpy_matrix(adj_matrix_gender,
                                   create_using=nx.DiGraph())
        fb100_df_tmp = []
        for j in nx.generate_edgelist(Gnx, data = False):
            fb100_df_tmp.append(map(np.int,j.split(' ')))
        fb100_df = pd.DataFrame(fb100_df_tmp)
        fb100_df.columns = ['from','to']
        fb100_df_undirect = fb100_df.copy()
        fb100_df_undirect['to'] = fb100_df['from']
        fb100_df_undirect['from'] = fb100_df['to']
        fb100_df = fb100_df.append(fb100_df_undirect)
        fb100_df['weight'] = 1
        fb100_df.head()
        fb100_df.drop_duplicates(inplace = True)
        fb100_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'.csv', header = False, index = False)
        y_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'_gender.csv', header = True, index = False)

## Increasing Homophily/Block Structure
## Across-Network Set-up
for iter_num in range(10):
    print('instance: '+str(iter_num))
    for block in np.array([1.1, 1.2, 1.3, 1.4, 1.5]):
        ( adj_matrix_gender, gender_y) = create_affiliation_model_temp(84,
                                              lambda_block_parameter =block,
                                              dispersion_parameter_vect = np.array([np.array([0, 0]),
                                                np.array([0, 0])]),
                                        class_size_vect = np.array([800,1200]))

        tag = str(iter_num) + '_Sloan_analog_SBM_across_network_' + str(block)
        
        gender_y = np.array(map(np.int,gender_y)) ## need np.int for machine precisions reasons
        print(np.mean(np.sum(adj_matrix_gender,1)))
        print(np.mean(np.sum(adj_matrix_gender,1)[gender_y==1]))
        print(np.mean(np.sum(adj_matrix_gender,1)[gender_y==0]))

        d = {'keys':range(len(gender_y)),
            'gender_y':gender_y}
        y_df = pd.DataFrame(d)
        Gnx = nx.from_numpy_matrix(adj_matrix_gender,
                                   create_using=nx.DiGraph())
        fb100_df_tmp = []
        for j in nx.generate_edgelist(Gnx, data = False):
            fb100_df_tmp.append(map(np.int,j.split(' ')))
        fb100_df = pd.DataFrame(fb100_df_tmp)
        fb100_df.columns = ['from','to']
        fb100_df_undirect = fb100_df.copy()
        fb100_df_undirect['to'] = fb100_df['from']
        fb100_df_undirect['from'] = fb100_df['to']
        fb100_df = fb100_df.append(fb100_df_undirect)
        fb100_df['weight'] = 1
        fb100_df.head()
        fb100_df.drop_duplicates(inplace = True)
        fb100_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'.csv', header = False, index = False)
        y_df.to_csv('../../data/SBM_ICWSM_revise/'+tag+'_gender.csv', header = True, index = False)

import glob
tmp = glob.glob("../../data/SBM_ICWSM_revise/*.csv")
col = []
for j in np.array(tmp):
    if 'gender' not in j:
        if 'across_network' in j:
            if 'Sloan' in j:
                j = re.sub('../../data/SBM_ICWSM_revise/', '', j)
                print("'"+j+"'")
                col.append(j) 
col<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.15111762550384755, 'fraction_numerical': 0.06310003664345915, 'mean_word_length': 2.7081521739130436, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 9, 'word_list_counts': {'cursed_substrings.json': 48, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '3', 'dir': 'jupyter-structured-clean-dedup', 'id': '553132', 'license': 'MIT', 'n_tokens_mistral': 5536, 'n_tokens_neox': 4835, 'n_words': 777, 'path': 'code/[ICWSM revision] Create_oSBM_for_ReFeX_features_simulation.ipynb', 'repo_name': 'kaltenburger/NetworkPredictionICWSM2021', 'size': '13645'}