<jupyter_start><jupyter_text>Read the graph and the data<jupyter_code>data_ab  = pd.read_csv("../../../generated_csvs/guncontrol_daily_polarities_base.csv")
graph_ab = nx.read_gexf("../../../graph/guncontrol_daily_largest.gexf")<jupyter_output><empty_output><jupyter_text>Divide the data into classes<jupyter_code># n_classes can be 2, 4, 6 or 8
n_classes = 2

# polarity_type can be either "Polarity Neighbours" or "Polarity Following"
polarity_type = "Polarity Neighbours"
base_polarities = data_ab[polarity_type].dropna()
base_polarities[base_polarities > 2.42]  = 2.42
base_polarities[base_polarities < -2.42] = -2.42

if n_classes == 2:
    base_polarities = base_polarities.to_frame()
    base_polarities['Class'] = 0
    base_polarities['Class'][base_polarities[polarity_type] >= 0] = 1
    
elif n_classes == 4:
    base_polarities = base_polarities.to_frame()
    base_polarities['Class'] = 0

    base_polarities['Class'][base_polarities[polarity_type] >= 0] = 1
    positive_quantiles = np.quantile(base_polarities[polarity_type][base_polarities[polarity_type] >= 0], q = 0.5)
    
    base_polarities['Class'][base_polarities[polarity_type] >= positive_quantiles] = 2

    base_polarities['Class'][base_polarities[polarity_type] < 0] = 3
    negative_quantiles = np.quantile(base_polarities[polarity_type][base_polarities[polarity_type] < 0], q = 0.5)
    
    base_polarities['Class'][base_polarities[polarity_type] <= negative_quantiles] = 4
    
    
elif n_classes == 6:
    base_polarities = base_polarities.to_frame()
    base_polarities['Class'] = 0

    base_polarities['Class'][base_polarities[polarity_type] >= 0] = 1
    positive_quantiles = np.quantile(base_polarities[polarity_type][base_polarities[polarity_type] >= 0], q = [0.33, 0.66])

    base_polarities['Class'][base_polarities[polarity_type] >= positive_quantiles[0]] = 2
    base_polarities['Class'][base_polarities[polarity_type] >= positive_quantiles[1]] = 3

    base_polarities['Class'][base_polarities[polarity_type] < 0] = 4
    negative_quantiles = np.quantile(base_polarities[polarity_type][base_polarities[polarity_type] < 0], q = [0.33, 0.66])

    base_polarities['Class'][base_polarities[polarity_type] <= negative_quantiles[1]] = 5
    base_polarities['Class'][base_polarities[polarity_type] <= negative_quantiles[0]] = 6
    
    
elif n_classes == 8:
    base_polarities = base_polarities.to_frame()
    base_polarities['Class'] = 0

    base_polarities['Class'][base_polarities[polarity_type] >= 0] = 1
    positive_quantiles = np.quantile(base_polarities[polarity_type][base_polarities[polarity_type] >= 0], q = [0.25, 0.5, 0.75])

    base_polarities['Class'][base_polarities[polarity_type] >= positive_quantiles[0]] = 2
    base_polarities['Class'][base_polarities[polarity_type] >= positive_quantiles[1]] = 3
    base_polarities['Class'][base_polarities[polarity_type] >= positive_quantiles[2]] = 4

    base_polarities['Class'][base_polarities[polarity_type] < 0] = 5
    negative_quantiles = np.quantile(base_polarities[polarity_type][base_polarities[polarity_type] < 0], q = [0.25, 0.5, 0.75])

    base_polarities['Class'][base_polarities[polarity_type] <= negative_quantiles[2]] = 6
    base_polarities['Class'][base_polarities[polarity_type] <= negative_quantiles[1]] = 7
    base_polarities['Class'][base_polarities[polarity_type] <= negative_quantiles[0]] = 8<jupyter_output>/home/adc/anaconda3/lib/python3.7/site-packages/ipykernel_launcher.py:8: SettingWithCopyWarning: 
A value is trying to be set on a copy of a slice from a DataFrame

See the caveats in the documentation: https://pandas.pydata.org/pandas-docs/stable/user_guide/indexing.html#returning-a-view-versus-a-copy<jupyter_text>Get the features<jupyter_code># relabel nodes as indexes
d  = dict(zip(data_ab['0'].values, data_ab['0'].index))
g2 = nx.relabel_nodes(graph_ab, d)<jupyter_output><empty_output><jupyter_text>Use the node degrees as features<jupyter_code>features_idx = np.load("../../../npy/guncontrol_daily/index.npy", allow_pickle = True)
features_mat = np.load("../../../npy/guncontrol_daily/node_features_pca.npy", allow_pickle = True)
map_nodes_to_features = {}
for idx, feature in zip(range(0, len(features_idx)), features_idx):
    map_nodes_to_features[d[feature]] = idx
    
for i in list(g2.nodes()):
    g2.nodes[i]['features'] = features_mat[map_nodes_to_features[i]]
    
for node in list(g2.nodes()):
    g2.node[node]['label'] = 'default'<jupyter_output><empty_output><jupyter_text>Alternative: use the eigenvectors as features<jupyter_code>with open("../../../pickle/guncontrol_daily_eig.pkl", 'rb') as _p:
    _, eigvec = pkl.load(_p)
    
map_nodes_to_eigvec = {}
for idx, node_name in zip(range(0, len(list(graph_ab.nodes()))), list(graph_ab.nodes())):
    map_nodes_to_eigvec[d[node_name]] = idx
    
for i in list(g2.nodes()):
    g2.nodes[i]['features'] = eigvec[map_nodes_to_eigvec[i], 1:]
    
for node in list(g2.nodes()):
    g2.node[node]['label'] = 'default'<jupyter_output><empty_output><jupyter_text>Alternative 2: use both the node degrees and the eigenvectors as features<jupyter_code>features_idx = np.load("../../../npy/guncontrol_daily/index.npy", allow_pickle = True)
features_mat = np.load("../../../npy/guncontrol_daily/node_features_pca.npy", allow_pickle = True)

map_nodes_to_features = {}
for idx, feature in zip(range(0, len(features_idx)), features_idx):
    map_nodes_to_features[d[feature]] = idx
    
with open("../../../pickle/guncontrol_daily_eig.pkl", 'rb') as _p:
    _, eigvec = pkl.load(_p)
    
map_nodes_to_eigvec = {}
for idx, node_name in zip(range(0, len(list(graph_ab.nodes()))), list(graph_ab.nodes())):
    map_nodes_to_eigvec[d[node_name]] = idx
    
for i in list(g2.nodes()):
    g2.nodes[i]['features'] = np.concatenate((features_mat[map_nodes_to_features[i]], eigvec[map_nodes_to_eigvec[i], 1:]))
    
for node in list(g2.nodes()):
    g2.node[node]['label'] = 'default'<jupyter_output><empty_output><jupyter_text>Train the model<jupyter_code># get the subgraph containing the known nodes
stellar = StellarGraph.from_networkx(g2, node_features = 'features')
subg    = stellar.subgraph(base_polarities['Class'].index)
train_labels, test_labels = model_selection.train_test_split(
    base_polarities['Class'],
    train_size=0.7,
    test_size=None,
    stratify=base_polarities['Class'],
    random_state=42,
)
val_labels, test_labels = model_selection.train_test_split(
    test_labels, train_size=0.2, test_size=None, stratify=test_labels, random_state=100,
)
target_encoding = preprocessing.LabelBinarizer()

train_targets = target_encoding.fit_transform(train_labels)
val_targets = target_encoding.transform(val_labels)
test_targets = target_encoding.transform(test_labels)

batch_size = 50
num_samples = [10, 10]

generator = GraphSAGENodeGenerator(subg, batch_size, num_samples)
train_gen = generator.flow(train_labels.index, train_targets, shuffle=True)

graphsage_model = GraphSAGE(
    layer_sizes=[32, 32], generator=generator, bias=True, dropout=0.5,
)

# choose the activation function and the loss based on the number of classes:
if n_classes == 2:
    x_inp, x_out = graphsage_model.in_out_tensors()
    prediction = layers.Dense(units=train_targets.shape[1], activation="sigmoid")(x_out)
    
    model = Model(inputs=x_inp, outputs=prediction)
    model.compile(
        optimizer=optimizers.Adam(lr=0.005),
        loss=losses.binary_crossentropy,
        metrics=["acc"],
    )
    
else:
    x_inp, x_out = graphsage_model.in_out_tensors()
    prediction = layers.Dense(units=train_targets.shape[1], activation="softmax")(x_out)
    
    model = Model(inputs=x_inp, outputs=prediction)
    model.compile(
        optimizer=optimizers.Adam(lr=0.005),
        loss=losses.categorical_crossentropy,
        metrics=["acc"],
    )
    
val_gen = generator.flow(val_labels.index, val_targets)
history = model.fit(
    train_gen, epochs=20, validation_data=val_gen, verbose=2, shuffle=False
)
# Save the train / validation training history
history_save_path = "../../../npy/classifier_history/guncontrol_daily/history_2_neighbours_node_degree.npy"
np.save(history_save_path, history.history)<jupyter_output><empty_output><jupyter_text>Test the model<jupyter_code>test_gen = generator.flow(test_labels.index, test_targets)

test_metrics = model.evaluate(test_gen)
print("\nTest Set Metrics:")
for name, val in zip(model.metrics_names, test_metrics):
    print("\t{}: {:0.4f}".format(name, val))<jupyter_output>370/370 [==============================] - 36s 97ms/step - loss: 0.1670 - acc: 0.9326

Test Set Metrics:
	loss: 0.1670
	acc: 0.9326<jupyter_text>Predict the unlabeled data<jupyter_code>generator = GraphSAGENodeGenerator(stellar, batch_size, num_samples)

hold_out_nodes = data_ab.index.difference(base_polarities.index)

data_ab['Class'] = 0

hold_out_targets = target_encoding.transform(data_ab.loc[hold_out_nodes]["Class"])
hold_out_gen = generator.flow(hold_out_nodes, hold_out_targets)

hold_out_predictions = model.predict(hold_out_gen)
hold_out_predictions = target_encoding.inverse_transform(hold_out_predictions)
pred          = data_ab
pred['Class'] = base_polarities['Class']
pred.loc[hold_out_nodes, 'Class'] = hold_out_predictions<jupyter_output><empty_output><jupyter_text>Save the model and the predictions<jupyter_code>model.save("../../../keras/guncontrol_daily/model")
pred[['0', 'Class']].to_csv("../../../keras/guncontrol_daily/predictions.csv", index = False)<jupyter_output>WARNING:tensorflow:From /home/adc/anaconda3/lib/python3.7/site-packages/tensorflow/python/ops/resource_variable_ops.py:1817: calling BaseResourceVariable.__init__ (from tensorflow.python.ops.resource_variable_ops) with constraint is deprecated and will be removed in a future version.
Instructions for updating:
If using Keras pass *_constraint arguments to layers.
INFO:tensorflow:Assets written to: ../../../keras/guncontrol_daily/model/assets<jupyter_text>TSNE visualization<jupyter_code>all_nodes  = pred['Class'].index
all_mapper = generator.flow(all_nodes)

embedding_model = Model(inputs = x_inp, outputs = x_out)
emb = embedding_model.predict(all_mapper)
# save the embeddings
np.save("../../../npy/embeddings/guncontrol_daily/emb.npy", emb)
emb_trans = TSNE(n_components = 2, n_jobs = -1).fit_transform(emb)
colours = []
for iter in range(0, len(map_nodes_to_features)):
    if pred.loc[map_nodes_to_features[iter], 'Class'] == 0: 
        colours.append('b')
    
    else:
        colours.append('r')
plt.figure(figsize = (20, 10))
plt.rcParams.update({'font.size': 22})
plt.scatter(emb_trans[:,0], emb_trans[:,1], c = colours, alpha = 0.5)
plt.tick_params(labelsize = 20)
plt.title("TSNE embedding for the 'gun control' data set")
plt.xlabel("Dimension 1")
plt.ylabel("Dimension 2")
plt.grid()
plt.show()
plt.figure(figsize = (20, 10))
plt.rcParams.update({'font.size': 22})
plt.scatter(emb_trans[:,0], emb_trans[:,1], c = colours, alpha = 0.5)
plt.tick_params(labelsize = 20)
plt.title("TSNE embedding for the 'gun control' data set")
plt.xlabel("Dimension 1")
plt.ylabel("Dimension 2")
plt.grid()

plt.savefig("../../../plots/guncontrol_daily/tsne_30_neighbours.png")
plt.close()<jupyter_output><empty_output><jupyter_text>Save map to features from node degree<jupyter_code>with open("../../../keras/guncontrol_daily/map_nodes_to_features.pkl", "wb") as _pkl:
    pkl.dump(map_nodes_to_features, _pkl)<jupyter_output><empty_output><jupyter_text>Save map to features from eigenvectors<jupyter_code>with open("../../../keras/guncontrol_daily/map_nodes_to_eigvec.pkl", "wb") as _pkl:
    pkl.dump(map_nodes_to_eigvec, _pkl)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.14380264741275572, 'fraction_numerical': 0.02037132542547705, 'mean_word_length': 5.78030303030303, 'pattern_counts': {'":': 0, '<': 63, '<?xml version=': 0, '>': 64, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '13', 'dir': 'jupyter-structured-clean-dedup', 'id': '4500', 'license': 'MIT', 'n_tokens_mistral': 4434, 'n_tokens_neox': 4155, 'n_words': 659, 'path': 'nets/notebooks/guncontrol_daily_classifier.ipynb', 'repo_name': 'andrei-comanescu/msc_thesis', 'size': '11634'}