<jupyter_start><jupyter_text>Load referral sites edges for level 1<jupyter_code>level = 1
referral_sites_NODES = get_referral_sites_edges(data_year=2018, level=level)

print(referral_sites_NODES[:5])
edge_df = pd.DataFrame(referral_sites_NODES, columns=['source', 'target'])

edge_df.head()<jupyter_output><empty_output><jupyter_text>Find all unique nodes in edges<jupyter_code>nodes_in_edges = list(set(edge_df.source.unique().tolist() + edge_df.target.unique().tolist()))
print('Number of unique nodes in edges:', len(nodes_in_edges), 'Sample:', nodes_in_edges[:5])<jupyter_output>Number of unique nodes in edges: 8250 Sample: ['anh.com', 'nrawomen.tv', 'evaluationtoolbox.net.au', 'guyhut.com', 'holodomorinfo.com']<jupyter_text>1. Load all node features<jupyter_code>node_features_df = load_node_features()
node_features_df = node_features_df.set_index('site')
node_features_df.head()<jupyter_output><empty_output><jupyter_text>Subset node_features<jupyter_code>node_features_df = node_features_df.loc[nodes_in_edges]
node_features_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 8250 entries, anh.com to animalmatters.org
Data columns (total 5 columns):
 #   Column                       Non-Null Count  Dtype  
---  ------                       --------------  -----  
 0   alexa_rank                   6466 non-null   float64
 1   daily_pageviews_per_visitor  6467 non-null   float64
 2   daily_time_on_site           4976 non-null   float64
 3   total_sites_linking_in       8110 non-null   float64
 4   bounce_rate                  4667 non-null   float64
dtypes: float64(5)
memory usage: 386.7+ KB<jupyter_text>2. Fill all missing alexa_rank and total_sites_linking_in with 0<jupyter_code>node_features_df.alexa_rank = node_features_df.alexa_rank.fillna(0)
node_features_df.total_sites_linking_in = node_features_df.total_sites_linking_in.fillna(0)
node_features_df.info()<jupyter_output><class 'pandas.core.frame.DataFrame'>
Index: 8250 entries, anh.com to animalmatters.org
Data columns (total 5 columns):
 #   Column                       Non-Null Count  Dtype  
---  ------                       --------------  -----  
 0   alexa_rank                   8250 non-null   float64
 1   daily_pageviews_per_visitor  6467 non-null   float64
 2   daily_time_on_site           4976 non-null   float64
 3   total_sites_linking_in       8250 non-null   float64
 4   bounce_rate                  4667 non-null   float64
dtypes: float64(5)
memory usage: 386.7+ KB<jupyter_text>3. Normalizing features<jupyter_code>import math

node_features_df['normalized_alexa_rank'] = node_features_df['alexa_rank'].apply(lambda x: 1/x if x else 0)
node_features_df['normalized_total_sites_linked_in'] = node_features_df['total_sites_linking_in'].apply(lambda x: math.log2(x) if x else 0)<jupyter_output><empty_output><jupyter_text>Create Graph<jupyter_code>import stellargraph as sg

G = sg.StellarGraph(nodes=node_features_df.loc[nodes_in_edges, ['normalized_alexa_rank', 'normalized_total_sites_linked_in']], edges=edge_df)

print(G.info())<jupyter_output>StellarGraph: Undirected multigraph
 Nodes: 8250, Edges: 15066

 Node types:
  default: [8250]
    Features: float32 vector, length 2
    Edge types: default-default->default

 Edge types:
    default-default->default: [15066]
        Weights: all 1 (default)
        Features: none<jupyter_text>Unsupervised GraphSAGEfrom stellargraph.mapper import GraphSAGELinkGeneratorfrom stellargraph.layer import GraphSAGE, link_classificationfrom stellargraph.data import UnsupervisedSamplerfrom tensorflow import keras<jupyter_code>from stellargraph.mapper import GraphSAGELinkGenerator
from stellargraph.layer import GraphSAGE, link_classification
from stellargraph.data import UnsupervisedSampler

from tensorflow import keras

# 1. Specify the other optional parameter values: root nodes, the number of walks to take per node, the length of each walk, and random seed.

nodes = list(G.nodes())
number_of_walks = 1
length = 5

# 2. Create the UnsupervisedSampler instance with the relevant parameters passed to it.
unsupervised_samples = UnsupervisedSampler(G, nodes=nodes, length=length, number_of_walks=number_of_walks)

# 3. Create a node pair generator:
batch_size = 128
epochs = 4
num_samples = [10, 5]

graphsage_link_generator = GraphSAGELinkGenerator(G, batch_size, num_samples)
train_graphsage_link_gen = graphsage_link_generator.flow(unsupervised_samples)

layer_sizes = [128, 512]
graphsage = GraphSAGE(
    layer_sizes=layer_sizes, generator=graphsage_link_generator, bias=True, dropout=0.0, normalize="l2"
)

# Build the model and expose input and output sockets of graphsage, for node pair inputs:
x_inp, x_out = graphsage.in_out_tensors()

prediction = link_classification(
    output_dim=1, output_act="sigmoid", edge_embedding_method="ip"
)(x_out)

graphsage_model = keras.Model(inputs=x_inp, outputs=prediction)

graphsage_model.compile(
    optimizer=keras.optimizers.Adam(lr=1e-3),
    loss=keras.losses.categorical_crossentropy,
    metrics=['acc'],
)
history = graphsage_model.fit(
    train_graphsage_link_gen,
    epochs=epochs,
    verbose=1,
    use_multiprocessing=False,
    workers=4,
    shuffle=True,
)
embedding_model = keras.Model(inputs=x_inp[0::2], outputs=x_out[0])

from stellargraph.mapper import GraphSAGENodeGenerator

node_gen = GraphSAGENodeGenerator(G, batch_size, num_samples).flow(node_features_df.index)
node_embeddings = embedding_model.predict(node_gen, workers=4, verbose=1)

embeddings_wv = dict(zip(node_features_df.index.tolist(), node_embeddings.tolist()))

# print('Sample:', embeddings_wv['crooked.com'][:10])<jupyter_output>65/65 [==============================] - 4s 58ms/step<jupyter_text>Export embeddings as feature<jupyter_code>export_model_as_feature(embeddings_wv, f'graph_sage_referral_sites_level_{level}_epochs_{epochs}', data_year='2018')
run_experiment(features=f'graph_sage_referral_sites_level_{level}_epochs_{epochs}', dataset='emnlp2018')
run_experiment(features=f'graph_sage_referral_sites_level_{level}_epochs_{epochs}', dataset='emnlp2018', task='bias')<jupyter_output>+------+-----------+---------------------+---------------+--------------------+--------------------------------------------+
| task |  dataset  | classification_mode | type_training | normalize_features |                  features                  |
+------+-----------+---------------------+---------------+--------------------+--------------------------------------------+
| bias | emnlp2018 |  single classifier  |    combine    |       False        | graph_sage_referral_sites_level_1_epochs_4 |
+------+-----------+---------------------+---------------+--------------------+--------------------------------------------+
{'fraction_non_alphanumeric': 0.1537433155080214, 'fraction_numerical': 0.029114676173499703, 'mean_word_length': 3.8058529621698787, 'pattern_counts': {'":': 0, '<': 33, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '9', 'dir': 'jupyter-structured-clean-dedup', 'id': '302643', 'license': 'MIT', 'n_tokens_mistral': 2344, 'n_tokens_neox': 2170, 'n_words': 410, 'path': 'notebooks/graph_algos/graphSage/corpus 2018 referral_sites lvl data 1.ipynb', 'repo_name': 'Panayot9/News-Media-Peers', 'size': '6732'}