<jupyter_start><jupyter_text>generate graph<jupyter_code>from graspy.simulations import sbm
import networkx as nx
sizes = [5,5]
probs = [[0.3, 0.01], [0.01,0.3]]
G1 = sbm(sizes,probs)
G2 = nx.stochastic_block_model(sizes, probs)
for (i,j) in G2.edges:   #these are all the edges indexed with idices of nodes
    print((i,j))
np.argwhere(G1>0)  #these are also the edges indexed with idices of nodes
G1 #G generated with graspy.simulations.sbm is essentially a weight matrix, 1 if connected, 0 if not connected
sizes = [10,10]
probs = [[0.5, 0.05], [0.05,0.5]]
g = nx.stochastic_block_model(sizes, probs, seed=0)
nx.draw(g,with_labels=True)
number_nodes=len(g)
number_nodes
E = len(g.edges)
E
cluster_sizes = [5,5]
probs = [[0.5, 0.2], [0.2, 0.5]]
g = nx.stochastic_block_model(cluster_sizes, probs)
node_ind = 3

weights = nx.simrank_similarity(g,source=node_ind,importance_factor=0.7)
weights<jupyter_output><empty_output><jupyter_text>Generate Data Points<jupyter_code>datapoints= {}
'''
datapoints: a dictionary containing the data of each node in the graph needed for the algorithm 1,
which are features, label, degree, and also the optimizer model for each node
'''
true_labels = []
'''
true_labels: the true labels for the nodes of the graph
'''
theta = np.array([[2, 2],
                  [-2,2]])
m=5
n=2
noise_sd = 0.1
cnt = 0
for i,cluster_size in enumerate(cluster_sizes):
    for j in range(cluster_size):
        
    
        features = np.random.normal(loc=0.0, scale=1.0, size=(m, n))
        '''
        features: the feature vector of node i which are i.i.d. realizations of a standard Gaussian random vector x~N(0,I)
        '''
        label = np.dot(features, theta[i]) + np.random.normal(0,noise_sd)
        '''
        label: the label of the node i that is generated according to the linear model y = x^T w + e
        '''

    #     true_labels.append(label)


    #     true_labels.append(labels)

        model = TorchLinearModel(n)
        '''
        model : the linear model for the node i that is implemented by pytorch
        '''
        optimizer = TorchLinearOptimizer(model)
        '''
        optimizer : the optimizer model for the node i that is implemented by pytorch with MSE loss function
        '''


        datapoints[cnt] = {
            'features': Variable(torch.from_numpy(features)).to(torch.float32),
            'model': model,
            'label': Variable(torch.from_numpy(label)).to(torch.float32),
            'optimizer': optimizer
        }
        cnt+=1
def generate_test_set():
    
    datapoints= {}
    '''
    datapoints: a dictionary containing the data of each node in the graph needed for the algorithm 1,
    which are features, label, degree, and also the optimizer model for each node.
    
    '''
    true_labels = []
    '''
    true_labels: the true labels for the nodes of the graph
    '''
    theta = np.array([[2, 2],
                      [-2,2]])
    m=5
    n=2
    noise_sd = 0.1
    cnt = 0
    for i,cluster_size in enumerate(cluster_sizes):
        for j in range(cluster_size):


            features = np.random.normal(loc=0.0, scale=1.0, size=(m, n))
            label = np.dot(features, theta[i]) + np.random.normal(0,noise_sd)



            datapoints[cnt] = {
                'features': features,
                'label': label
            }
            cnt+=1
    return datapoints
datapoints[9]<jupyter_output><empty_output><jupyter_text>train a local model<jupyter_code>model = datapoints[node_ind]['model']
weights = nx.simrank_similarity(g,source=node_ind,importance_factor=0.3)
optimizer = torch.optim.RMSprop(model.parameters())
for i in range(300):
    optimizer.zero_grad()
    loss=0
    for i in range(10):
        w = weights[i]
        X = datapoints[i]['features']
        pred_y = model(X)
        y = datapoints[i]['label'].reshape(-1,1)
        loss+= w*torch.nn.MSELoss(reduction='mean')(pred_y,y)
    loss.backward()
    optimizer.step()

model.linear.weight.data<jupyter_output><empty_output><jupyter_text>define models, optimizers, penalties<jupyter_code>import torch
import abc
import torch.nn.functional as F

from abc import ABC


# The linear model which implemented by pytorch
class TorchLinearModel(torch.nn.Module):
    def __init__(self, n):
        super(TorchLinearModel, self).__init__()
        self.linear = torch.nn.Linear(n, 1, bias=False)

    def forward(self, x):
        y_pred = self.linear(x)
        return y_pred
    
    
# The abstract optimizer model which should have model, optimizer, and criterion
class Optimizer(ABC):
    def __init__(self, model, optimizer, criterion):
        self.model = model
        self.optimizer = optimizer
        self.criterion = criterion

    @abc.abstractmethod
    def optimize(self, x_data, y_data, old_weight, regularizer_term):
        torch_old_weight = torch.from_numpy(np.array(old_weight, dtype=np.float32))
        self.model.linear.weight.data = torch_old_weight
        for iterinner in range(5):
            self.optimizer.zero_grad()
            y_pred = self.model(x_data)
            loss1 = self.criterion(y_pred, y_data)
            loss2 = 1 / (2 * regularizer_term) * torch.mean((self.model.linear.weight - torch_old_weight) ** 2) 
            loss = loss1 + loss2
            loss.backward()
            self.optimizer.step()

        return self.model.linear.weight.data.numpy()
    
    
# The Linear optimizer model which implemented by pytorch    
class TorchLinearOptimizer(Optimizer):
    def __init__(self, model):
        criterion = torch.nn.MSELoss(reduction='mean')
        optimizer = torch.optim.RMSprop(model.parameters())
        super(TorchLinearOptimizer, self).__init__(model, optimizer, criterion)

    def optimize(self, x_data, y_data, old_weight, regularizer_term):
        torch_old_weight = torch.from_numpy(np.array(old_weight, dtype=np.float32))
        self.model.linear.weight.data = torch_old_weight
        for iterinner in range(5):
            self.optimizer.zero_grad()
            y_pred = self.model(x_data)
            loss1 = self.criterion(y_pred, y_data)
            loss2 = 1 / (2 * regularizer_term) * torch.mean((self.model.linear.weight - torch_old_weight) ** 2) 
            loss = loss1 + loss2
            loss.backward()
            self.optimizer.step()

        return self.model.linear.weight.data.numpy()
# The abstract penalty function which has a function update
class Penalty(ABC):
    def __init__(self, lambda_lasso, weight_vec, Sigma, n):
        self.lambda_lasso = lambda_lasso
        self.weight_vec = weight_vec
        self.Sigma = Sigma

    @abc.abstractmethod
    def update(self, new_u):
        pass


# The norm2 penalty function
class Norm2Pelanty(Penalty):
    def __init__(self, lambda_lasso, weight_vec, Sigma, n):
        super(Norm2Pelanty, self).__init__(lambda_lasso, weight_vec, Sigma, n)
        self.limit = np.array(lambda_lasso * weight_vec)

    def update(self, new_u):
        normalized_u = np.where(np.linalg.norm(new_u, axis=1) >= self.limit)
        new_u[normalized_u] = (new_u[normalized_u].T * self.limit[normalized_u] / np.linalg.norm(new_u[normalized_u], axis=1)).T
        return new_u


# The MOCHA penalty function
class MOCHAPelanty(Penalty):
    def __init__(self, lambda_lasso, weight_vec, Sigma, n):
        super(MOCHAPelanty, self).__init__(lambda_lasso, weight_vec, Sigma, n)
        self.normalize_factor = 1 + np.dot(2 * self.Sigma, 1/(self.lambda_lasso * self.weight_vec))

    def update(self, new_u):
        for i in range(new_u.shape[1]):
            new_u[:, i] /= self.normalize_factor

        return new_u

    
# The norm1 penalty function
class Norm1Pelanty(Penalty):
    def __init__(self, lambda_lasso, weight_vec, Sigma, n):
        super(Norm1Pelanty, self).__init__(lambda_lasso, weight_vec, Sigma, n)
        self.limit = np.array([np.zeros(n) for i in range(len(weight_vec))])
        for i in range(n):
            self.limit[:, i] = lambda_lasso * weight_vec

    def update(self, new_u):
        normalized_u = np.where(abs(new_u) >= self.limit)
        new_u[normalized_u] = self.limit[normalized_u] * new_u[normalized_u] / abs(new_u[normalized_u])
        return new_u<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.11475808412351082, 'fraction_numerical': 0.015681983953318747, 'mean_word_length': 2.942022041207475, 'pattern_counts': {'":': 0, '<': 17, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '4', 'dir': 'jupyter-structured-clean-dedup', 'id': '266938', 'license': 'MIT', 'n_tokens_mistral': 2855, 'n_tokens_neox': 2755, 'n_words': 653, 'path': 'LWR_SBM.ipynb', 'repo_name': 'YuTian8328/flow-based-clustering', 'size': '8226'}