<jupyter_start><jupyter_text>=========================================================== Dataset visualization ===========================================================  Last update: 09 Dec 2019  Imports<jupyter_code>from __future__ import division, print_function, absolute_import, unicode_literals

import scipy as sp
import pandas as pd
import numpy as np
import tensorflow as tf
import matplotlib.pyplot as plt
import seaborn as sns
from collections import defaultdict
import sys
from sklearn.model_selection import KFold
from sklearn.model_selection import train_test_split

from tensorflow.keras.models import Sequential
from tensorflow.keras.layers import Dense, Activation, Dropout, Conv1D, LSTM, MaxPooling1D, GlobalAveragePooling1D
from tensorflow.keras import regularizers

sys.path.insert(0, '../src/')
%matplotlib inline

import utils
from mytimer import Timer<jupyter_output>/home/omid/.local/lib/python3.5/site-packages/h5py/__init__.py:36: FutureWarning: Conversion of the second argument of issubdtype from `float` to `np.floating` is deprecated. In future, it will be treated as `np.float64 == np.dtype(float).type`.
  from ._conv import register_converters as _register_converters<jupyter_text>Parameters<jupyter_code>data_fpath = '/home/omid/Datasets/Jeopardy/supervised_data.pk'
test_fraction = 0.2
runs = 30<jupyter_output><empty_output><jupyter_text>Helper functions<jupyter_code>def get_normalized_influence_on_others(A: np.ndarray, index: int) -> float:
    A = np.array(A)
    return sum(
        A[[i for i in range(len(A))
           if i != index], index])<jupyter_output><empty_output><jupyter_text>Loading the data<jupyter_code>data = utils.load_it(data_fpath)
print(len(data['y']))
mats = []
for i in range(len(data['y'])):
    mats.append(data['y'][i]['influence_matrix'])
average_matrix = np.mean(mats, axis=0)
sns.heatmap(average_matrix);<jupyter_output><empty_output><jupyter_text>Message: People mostly believe in themselves and are influence by themselves than others.<jupyter_code>std_matrix = np.std(mats, axis=0)
sns.heatmap(std_matrix);
all_influence_items = []
for matrix in mats:
    all_influence_items.extend(np.array(matrix.flatten())[0])
all_influence_items = np.array(all_influence_items)
plt.hist(all_influence_items);
len(np.where(all_influence_items == 0.25)[0]) / len(all_influence_items)<jupyter_output><empty_output><jupyter_text>Message: most common influence value is 0.25. If we only say 0.25 to every item in the matrix, we are exactly correct by 45.77% of time.  Are there always one person who is the most influential or multiple people?<jupyter_code>nums = []
for matrix in mats:
    how_influential_one_is = [get_normalized_influence_on_others(matrix, index) for index in range(4)]
#     np.argmax(how_influential_one_is)
    nums.append(
        len(np.where(how_influential_one_is == max(how_influential_one_is))[0]))
plt.hist(nums);<jupyter_output><empty_output><jupyter_text>Message: It is good. Mostly there is one person who is the most influential and not multiple people.<jupyter_code># Who is the most influential?
infl = []
for matrix in mats:
    how_influential_one_is = [get_normalized_influence_on_others(matrix, index) for index in range(4)]
    infl.append(np.argmax(how_influential_one_is))
plt.hist(infl);<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.13261000602772754, 'fraction_numerical': 0.01235684147076552, 'mean_word_length': 6.629885057471264, 'pattern_counts': {'":': 0, '<': 28, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '323773', 'license': 'MIT', 'n_tokens_mistral': 1153, 'n_tokens_neox': 1102, 'n_words': 265, 'path': 'notebooks/2.b- Dataset_visualization.ipynb', 'repo_name': 'omid55/appraisal_network_dynamics', 'size': '3318'}