<jupyter_start><jupyter_text>**Convert to edge-based graph**<jupyter_code>import networkx as nx
L = nx.line_graph(nx.DiGraph(g_partition))
nodes = list(L.nodes())
g_partition[nodes[10][0]][nodes[10][1]]['data']<jupyter_output><empty_output><jupyter_text>**Extract dynamic (speed) + static features from nodes**<jupyter_code>from sklearn.preprocessing import OneHotEncoder, StandardScaler, OrdinalEncoder
enc = OneHotEncoder(handle_unknown='ignore')
ienc = OrdinalEncoder()
scaler = StandardScaler()
def arc_features(arc):
    arc = g_partition[arc[0]][arc[1]]
    return [
        arc['data'].metadata['highway'],
        arc['data'].metadata.get('surface', 'no_sur'),
        arc['data'].roadClass.name
    ],  [float(arc['data'].metadata.get('maxspeed', '50')), 
        int(arc['data'].metadata.get('lanes', '1'))]
def construct_features():
    data = list()
    data_ord = list()
    for node in L.nodes:
        data.append(arc_features(node)[0])
        data_ord.append(arc_features(node)[1])
    return enc.fit_transform(data), ienc.fit_transform(data_ord)
    
x, y = construct_features()
  
enc.categories_
ienc.categories_
x.shape
x<jupyter_output><empty_output><jupyter_text>**Preprocess adjacency matrix**<jupyter_code>adj = nx.to_scipy_sparse_matrix(L, format="coo")
import scipy.sparse as sp
import numpy as np
import torch

def sparse_mx_to_torch_sparse_tensor(sparse_mx):
    """Convert a scipy sparse matrix to a torch sparse tensor."""
    sparse_mx = sparse_mx.tocoo().astype(np.float32)
    indices = torch.from_numpy(
        np.vstack((sparse_mx.row, sparse_mx.col)).astype(np.int64))
    values = torch.from_numpy(sparse_mx.data)
    shape = torch.Size(sparse_mx.shape)
    return torch.sparse.FloatTensor(indices, values, shape)
                                    
def normalize(mx):
    """Row-normalize sparse matrix"""
    rowsum = np.array(mx.sum(1))
    
    r_inv = np.power(rowsum, -1).flatten()
    r_inv[np.isinf(r_inv)] = 0.
    r_mat_inv = sp.diags(r_inv)
    mx = r_mat_inv.dot(mx)
    return mx

# build symmetric adjacency matrix
adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)
                                    
adj = normalize(adj + sp.eye(adj.shape[0]))
adj = sparse_mx_to_torch_sparse_tensor(adj)
                                    
# adj = nx.to_scipy_sparse_matrix(L, format="coo")
# adj = adj + adj.T.multiply(adj.T > adj) - adj.multiply(adj.T > adj)
# def diagonal_degree_matrix(adj):
#     diag = np.zeros([adj.shape[0], adj.shape[0]]) # basically dimensions of your graph
#     rows, cols = adj.nonzero()
#     for row, col in zip(rows, cols):
#         diag[row, row] += 1
#     return diag

# D = diagonal_degree_matrix(adj)
# np.power(D, -0.5).dot(adj).dot(np.power(D, -0.5))
#Our speed data uses segment ids, but the model uses sequential indexes, based on `.nodes()`
import math
id_to_idx = {}
# defaultdict won't do what you expect in Pandas
df = pandas.read_csv("../data/timeseries_speed_april_first_week.csv")
df = df.T
l = (df.isnull().mean() < 0.5).tolist()

indices = [i for i, x in enumerate(l) if x == True]
print(indices)
TOTAL_T_STEPS = 144
id_to_idx = {}

for idx, id_ in enumerate(L.nodes()):
    id_to_idx[id_] = idx
df = df.T
df = df.loc[:, df.columns != 'Unnamed: 0']

df2 = df['from_node']
df3 = df['to_node']

df_filled = df.loc[:, df.columns != 'from_node']
df_filled = df.loc[:, df.columns != 'to_node']


#df_filled = df_filled.interpolate(method='nearest', axis=1)
SPEED_COLUMNS = list(map(str, range(TOTAL_T_STEPS)))
df_filled = df_filled.fillna(df_filled[SPEED_COLUMNS].mean())
df_filled['from_node'] = df2
df_filled['to_node'] = df3

print(df_filled[0:10])

df_filled[0:10]<jupyter_output><empty_output><jupyter_text>**Create rolling window tensor dataset**<jupyter_code>import torch
import scipy.sparse
TOTAL_T_STEPS = 144

from sklearn.preprocessing import OneHotEncoder, OrdinalEncoder, RobustScaler
enc = OneHotEncoder(handle_unknown='ignore')
ienc = OrdinalEncoder()
scaler = RobustScaler()
import math
  
speed_features = df_filled.values.flatten()
speed_features = np.array([s for s in speed_features if not math.isnan(s)]).reshape(-1,1)
print(speed_features[0:10])  
scaler.fit(speed_features)
scaler.transform(np.array(speed_features[0:10]).reshape(-1,1))
def build_dataset_to_numpy_tensor(from_=0, to=TOTAL_T_STEPS, df=None, norm=False):
    """
    We extract features from speed (actual speed, whether speed is missing)
    and combine with static features.
    :return:
         np.ndarray: dataset tensor of shape [num_time_steps, num_nodes, num_features]
    """
    dataset = list()
    for t in range(from_, to):
        cat_features_at_t = [['primary', 'asphalt', 'MajorRoad']] * len(L.nodes)
        ord_features_at_t = [[50.0, 4]] * len(L.nodes)
        speed_features_at_t = [50] * len(L.nodes) 
        speed_is_nan_feature = [1] * len(L.nodes)
        for _, row in df.iterrows():

            arc = (row['from_node'], row['to_node'])
            cat_features_at_t[id_to_idx[arc]], ord_features_at_t[id_to_idx[arc]]  = arc_features(arc)
            speed_features_at_t[id_to_idx[arc]] = row[str(t)]
            if np.isnan(row[str(t)]): 
                speed_is_nan_feature[id_to_idx[arc]] = 0
        dataset.append(np.concatenate([scaler.transform(np.array(speed_features_at_t).reshape(-1, 1)) if norm else  np.array(speed_features_at_t).reshape(-1, 1), 
                                       np.array(speed_is_nan_feature).reshape(-1, 1), 
                                       ienc.fit_transform(ord_features_at_t),
                                       enc.fit_transform(cat_features_at_t).toarray()], axis=1))
    return np.stack(dataset, axis=0)

Y = build_dataset_to_numpy_tensor(df=df)
Y_filled = build_dataset_to_numpy_tensor(df=df_filled, norm=True)
X = np.moveaxis(Y, source=(0,1,2), destination=(2,0,1))
X_filled = np.moveaxis(Y_filled, source=(0,1,2), destination=(2,0,1))


# num_vertices, num_features, num_timesteps
X.shape
# Build mask tensor
X_masked = torch.where(torch.isnan(torch.from_numpy(X)), torch.tensor([0]), torch.tensor([1]))
X_masked = X_masked.bool()
print(X.shape)
print(X_masked.shape)
def generate_dataset_concat(X, X_masked, num_timesteps_input, num_timesteps_output):
    """
    Takes node features for the graph and divides them into multiple samples
    along the time-axis by sliding a window of size (num_timesteps_input+
    num_timesteps_output) across it in steps of 1.
    :param X: Node features of shape (num_vertices, num_features,
    num_timesteps)
    :return:
        - Node data (features + labels) divided into multiple samples. Shape is
          (num_samples, num_vertices, num_features, num_timesteps_input).
        
    """
    # Generate the beginning index and the ending index of a sample, which
    # contains (num_points_for_training + num_points_for_predicting) points
    indices = [(i, i + (num_timesteps_input + num_timesteps_output)) for i
               in range(X.shape[2] - (
                num_timesteps_input + num_timesteps_output) + 1)]

    # Save samples
    features, target = [], []
    mask = []
    for i, j in indices:
        features.append(X[:, :, i: i + num_timesteps_input])
        target.append(X[:, 0, i + num_timesteps_input: j])
        mask.append(X_masked[:, 0, i + num_timesteps_input: j])

    return torch.from_numpy(np.array(features)), torch.from_numpy(np.array(target)),torch.stack(mask)
# training, validation, testing : 0.7, 0.1, 0.2
split_line1 = int(X.shape[2] * 0.7)
split_line2 = int(X.shape[2] * 0.9)
train_original_data = X_filled[:, :, :split_line1]
val_original_data = X_filled[:, :, split_line1:split_line2]
test_original_data = X_filled[:, :, split_line2:]

train_mask = X_masked[:, :, :split_line1]
valid_mask = X_masked[:, :, split_line1:split_line2]
test_mask = X_masked[:, :, split_line2:]


look_back = 9
look_ahead = 1
# num_samples, num_nodes, num_timesteps, num_features

training_data, training_target, train_mask = generate_dataset_concat(train_original_data, train_mask,
                                                       num_timesteps_input=look_back,
                                                       num_timesteps_output=look_ahead)
valid_data, valid_target, valid_mask = generate_dataset_concat(val_original_data, valid_mask,
                                             num_timesteps_input=look_back,
                                             num_timesteps_output=look_ahead)
test_data, test_target, test_mask = generate_dataset_concat(test_original_data, test_mask,
                                               num_timesteps_input=look_back,
                                               num_timesteps_output=look_ahead)

print(f"shape of training: {training_data.shape}, {training_target.shape}, {train_mask.shape}")
print(f"shape of validation: {valid_data.shape}, {valid_target.shape}, {valid_mask.shape}")
print(f"shape of testing: {test_data.shape}, {test_target.shape}, {test_mask.shape}")
train_mask[1, :, :].shape
from test_tube import Experiment 
from pytorch_lightning import Trainer
import os

# PyTorch summarywriter with a few bells and whistles    
exp = Experiment(save_dir=os.getcwd())

# pass in experiment for automatic tensorboard logging.    
trainer = Trainer(experiment=exp, max_nb_epochs=45, train_percent_check=1)
import h5py
with h5py.File('data.hdf5', 'w') as f:
    f.create_dataset("train", data=training_data)
    f.create_dataset("valid", data=valid_data)
    f.create_dataset("test", data=test_data)
with h5py.File('target.hdf5', 'w') as f:
    f.create_dataset("train", data=training_target)
    f.create_dataset("valid", data=valid_target)
    f.create_dataset("test", data=test_target)
with h5py.File('mask.hdf5', 'w') as f:
    f.create_dataset("train", data=train_mask)
    f.create_dataset("valid", data=valid_mask)
    f.create_dataset("test", data=test_mask)
type(test_data)
data = {}
data['train'] = training_data
data['valid'] = valid_data
data['test'] = test_data

target = {}
target['train'] = training_target
target['valid'] = valid_target
target['test'] = test_target

mask = {}
mask['train'] = train_mask
mask['valid'] = valid_mask
mask['test'] = test_mask
# batch shape: torch.Size([1, 6163, 26, 10])
print(valid_data.shape)
print(valid_mask[10, :, :])
print(valid_target[10, :, :])

# IMPORTANT: we normalize speed
print(valid_target[10, :, :].masked_select(valid_mask[10, :, :]))<jupyter_output>torch.Size([20, 6163, 29, 9])
tensor([[ True],
        [ True],
        [False],
        ...,
        [False],
        [False],
        [False]])
tensor([[1.7845],
        [1.7845],
        [1.1043],
        ...,
        [1.1043],
        [1.1043],
        [1.1043]], dtype=torch.float64)
tensor([ 1.7845e+00,  1.7845e+00,  2.0257e+00,  2.0257e+00,  1.7845e+00,
         1.7845e+00,  2.2493e+00,  2.2178e+00,  2.1416e+00,  2.0101e+00,
        -2.8412e+00,  2.2807e+00,  2.2807e+00,  2.0406e+00,  2.0406e+00,
         2.1416e+00,  2.1416e+00,  2.1416e+00,  2.0257e+00,  2.2178e+00,
         2.2619e+00,  1.7970e+00,  1.7970e+00,  2.1756e+00,  2.1756e+00,
         1.7970e+00,  1.7970e+00,  2.3061e+00,  2.3061e+00,  2.0881e+00,
         2.0881e+00,  2.0557e+00,  2.1464e+00,  2.1039e+00, -3.9616e+00,
        -3.1079e+00,  1.9234e+00, -2.8746e+00,  2.2798e+00,  2.0426e+00,
         1.8834e+00,  2.1416e+00,  2.1416e+00,  1.9875e+00,  1.9982e+00,
         1.9875e+00,  2.1416e+00,  2.1416e+00,  1.8548[...]<jupyter_text>**Now start training**<jupyter_code>from torch.utils.data import DataLoader, TensorDataset, ConcatDataset, ChainDataset
target['valid'].shape
adj.shape
a = torch.split(data['valid'], 4000, dim=1)[0].permute(1,0,2,3)
b = torch.split(target['valid'], 4000, dim=1)[0].permute(1,0,2)
datasets = ConcatDataset((TensorDataset(data['train'].permute(1,0,2,3), target['train'].permute(1,0,2)), TensorDataset(a,b)))
from torch.utils.data import Sampler
class CustomTensorDataset(TensorDataset):
    def __init__(self, *tensors, adj_tensor):
        assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
        self.tensors = tensors
        self.adj_tensor = adj_tensor

    def __getitem__(self, index):
        return tuple((tensor[index], self.adj_tensor) for tensor in self.tensors)

    def __len__(self):
        return self.tensors[0].size(0)
    
class CustomSampler(Sampler):

    def __init__(self, data_source, cum_indices, shuffle=True):
        super().__init__(data_source)
        self.data_source = data_source
        self.cum_indices = [0] + cum_indices
        self.shuffle = shuffle

    def __iter__(self):
        batch = []
        for prev, curr in zip(self.cum_indices, self.cum_indices[1:]):
            for idx in range(prev, curr):
                batch.append(idx)
            yield batch
            batch = []

    def __len__(self):
        return len(self.data_source)
adjs = [adj, adj1]
dl = DataLoader(datasets, batch_sampler=CustomSampler(datasets, datasets.cumulative_sizes), shuffle=False)
batches = list()
for batch_nb, batch in enumerate(dl):
    x, y = batch

    print(f'batch_nb: {batch_nb}, adj: {adjs[batch_nb].shape}')
    batches.append(CustomTensorDataset(x, y, adj_tensor=adjs[batch_nb]))
dl = DataLoader(ConcatDataset(batches), batch_size=1, shuffle=False)
def to_sparse(x):
    """ converts dense tensor x to sparse format """
    x_typename = torch.typename(x).split('.')[-1]
    sparse_tensortype = getattr(torch.sparse, x_typename)

    indices = torch.nonzero(x)
    if len(indices.shape) == 0:  # if all elements are zeros
        return sparse_tensortype(*x.shape)
    indices = indices.t()
    values = x[tuple(indices[i] for i in range(indices.shape[0]))]
    return sparse_tensortype(indices, values, x.size())
for batch_nb, data_batch in enumerate(dl):
    (x, _adj), (y, _) = data_batch
    _adj = to_sparse(_adj.to_dense().squeeze(dim=0))
    print(_adj.shape)



d2 = TensorDataset(a,b, adj1)
from random import sample
l1 = nx.line_graph(nx.DiGraph(g_partition))
l1 = sample(L.nodes(), 4000)
L1 = nx.line_graph(nx.DiGraph(l1))
adj1 = nx.to_scipy_sparse_matrix(L1, format="coo")

# build symmetric adjacency matrix
adj1 = adj1 + adj1.T.multiply(adj1.T > adj1) - adj1.multiply(adj1.T > adj1)
                                    
adj1 = normalize(adj1 + sp.eye(adj1.shape[0]))
adj1 = sparse_mx_to_torch_sparse_tensor(adj1)
print(adj1.shape)
import imp  
import src.models.tgcn.temporal_spatial_model as l
import src.models.tgcn.layers.lstmcell as h
h = imp.reload(h)

l = imp.reload(l)
TGCN = l.TGCN
model = TGCN(input_dim=29, hidden_dim=29, layer_dim=2, output_dim=1, adjs=(adj, adj1), 
             datasets=data, targets= target, mask=mask, scaler=scaler)
trainer.fit(model)
import imp  
import src.tgcn.stgcn as l

# PyTorch summarywriter with a few bells and whistles    
exp = Experiment(save_dir="../data")

# pass in experiment for automatic tensorboard logging.    
trainer = Trainer(experiment=exp, max_nb_epochs=30, train_percent_check=1)

l = imp.reload(l)
STGCN = l.STGCN
model2 = STGCN(adj=adj, datasets=data, targets=target, mask=mask, scaler=scaler)
trainer.fit(model2)
import imp  
import src.tgcn.layers.lstm as l

# PyTorch summarywriter with a few bells and whistles    
exp = Experiment(save_dir="../data")

# pass in experiment for automatic tensorboard logging.    
trainer = Trainer(experiment=exp, max_nb_epochs=30, train_percent_check=1)

l = imp.reload(l)
LSTMs = l.LSTMs
model3 = LSTMs(input_dim=9, datasets=data, targets=target, mask=mask)
trainer.fit(model3)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.139194847020934, 'fraction_numerical': 0.03942028985507246, 'mean_word_length': 3.670878459687124, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 19, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '5', 'dir': 'jupyter-structured-clean-dedup', 'id': '621320', 'license': 'Apache-2.0', 'n_tokens_mistral': 5942, 'n_tokens_neox': 5641, 'n_words': 1083, 'path': 'notebook/TGCN.ipynb', 'repo_name': 'tumeteor/ST-GCN', 'size': '15525'}