<jupyter_start><jupyter_text>AI-based Networking ジュニアチャレンジ サンプルプログラム<jupyter_code>!pip install torch pandas numpy joblib tqdm sklearn 
import json
import glob
import os
import torch
import torch.nn as nn
import torch.optim as optim
from torch.autograd import Variable
import pandas as pd
import numpy as np
import joblib
from tqdm.notebook import tqdm
from torch.utils.data import Dataset, DataLoader, Subset
from sklearn.model_selection import KFold, train_test_split
from sklearn.preprocessing import MinMaxScaler, StandardScaler
class Transformer:
    def __init__(self, is_train=False):
        self.transformer = dict()
        self.is_train = is_train
        if not self.is_train:
            self.transformer = self.load()

    def __call__(self, df, metric):
        if self.is_train:
            return self.fit_transform(df, metric)
        else:
            df[:] = self.transformer[metric].transform(df)
            return df

    def inverse_transform(self, scaled, metric):
        return self.transformer[metric].inverse_transform(scaled)

    def fit_transform(self, df, metric):
        self.transformer[metric] = MinMaxScaler()
        df = self.transformer[metric].fit_transform(df)
        self.dump()
        return df

    def dump(self, filename='/tmp/transformer.bin'):
        with open(filename, 'wb') as f:
            joblib.dump(self.transformer, f)

    def load(self, filename='/tmp/transformer.bin'):
        with open(filename, 'rb') as f:
            data = joblib.load(f)
        return data
class SimilarityDataset(Dataset):
    def __init__(self, path, sequence_length, device=None, transform=None, is_train=True):
        self.sequence_length = sequence_length
        self.transform = transform

        if device is None: self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
        else:
            self.device = device

        metrics        = ["psnr", "ssim"]
        target_metrics = ["throughput", "loss_rate"]

        self.input_dim  = len(metrics)
        self.target_dim = len(target_metrics)

        if is_train:
            files = glob.glob(os.path.join(path, "*.json"))
            df = self.read_files(files, self.sequence_length)
        else:
            df = self.read_file(path, sequence_length)

        if self.transform:
            for metric in metrics:
                df[[metric]] = self.transform(df[[metric]], metric)

        df.index = df[["video_type", "throughput", "loss_rate", "interval"]]
        indices = df[["video_type", "throughput", "loss_rate", "interval"]].index.unique()
        grouped_df = df.groupby(["video_type", "throughput", "loss_rate", "interval"])

        self.data = []
        self.target = []
        for index in tqdm(indices):
            series = grouped_df.get_group(index)
            if len(series) < self.sequence_length:
                continue
            video_type, throughput, loss_rate, interval = index
            self.data.append(series[metrics].values)
            self.target.append([throughput, loss_rate])

    def __getitem__(self, idx):
        ret = torch.tensor(self.data[idx], dtype=torch.float64, device=self.device)
        trg = torch.tensor(self.target[idx], dtype=torch.float64, device=self.device)
        return ret, trg

    def __len__(self):
        return len(self.data)

    def read_files(self, files, sequence_length):
        li_df = []
        for filename in tqdm(files):
            df = self.read_file(filename, sequence_length)
            li_df.append(df)
        df = pd.concat(li_df)
        return df

    def read_file(self, filename, sequence_length):
        with open(filename) as f:
            d = json.load(f)
        df = pd.DataFrame.from_dict(d)
        df["interval"] = df["frame_index"] // sequence_length
        df = df.sort_values("frame_index")
        return df
def collate_fn(batch):
    data, targets = list(zip(*batch))
    data    = torch.stack(data)
    targets = torch.stack(targets)
    return data, targets

def set_seed(seed):
    np.random.seed(seed)
    torch.manual_seed(seed)
class NetworkStateEstimationFromVideoStreaming(nn.Module):
    def __init__(self, input_dim, target_dim, sequence_length, hidden_dim):
        super(NetworkStateEstimationFromVideoStreaming, self).__init__()
        self.input_dim       = input_dim
        self.target_dim      = target_dim
        self.sequence_length = sequence_length
        self.hidden_dim      = hidden_dim

        self.fc1     = nn.Linear(self.input_dim * self.sequence_length, self.hidden_dim)
        self.fc2     = nn.Linear(self.hidden_dim, self.target_dim)
        self.relu    = nn.ReLU()

    def forward(self, x):
        x = x.view(-1, self.input_dim * self.sequence_length)
        out = self.fc1(x)
        out = self.relu(out)
        out = self.fc2(out)
        return out
def train(input_dir="/content/drive/MyDrive/similarity_measures/train", model_dir="/content/models", seed=1, fold=4, batchsize=32, log_seq=1):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
    transformer = Transformer(is_train=True)

    similarity_dir = input_dir
    model_dir      = model_dir
    seed           = seed
    fold_split     = fold
    batch_size     = batchsize

    set_seed(seed)

    sequence_length = 1000
    hidden_dim      = 128
    num_layers      = 2
    max_epoches     = 1000
    log_seq         = log_seq

    train_and_valid_dataset = SimilarityDataset(
            path=similarity_dir,
            sequence_length=sequence_length,
            device=device,
            transform=None
            )
    if not os.path.exists(model_dir):
        os.makedirs(model_dir)

    X = train_and_valid_dataset.data
    Y = train_and_valid_dataset.target
    skf = KFold(n_splits=fold_split, shuffle=True, random_state=seed)
    for i, (train_idx, valid_idx) in enumerate(skf.split(X, Y)):
        train_dataset = Subset(train_and_valid_dataset, train_idx)
        valid_dataset = Subset(train_and_valid_dataset, valid_idx)
        train_size = len(train_dataset)
        valid_size = len(valid_dataset)
        print(f'fold : {i+1} train dataset size : {train_size} valid dataset size: {valid_size}')

        train_dataloader = DataLoader(train_dataset, batch_size=batch_size, shuffle=True, collate_fn=collate_fn)
        valid_dataloader = DataLoader(valid_dataset, batch_size=valid_size, shuffle=True, collate_fn=collate_fn)

        input_dim  = train_and_valid_dataset.input_dim
        target_dim = train_and_valid_dataset.target_dim

        model = NetworkStateEstimationFromVideoStreaming(input_dim, target_dim, sequence_length, hidden_dim).to(device)

        loss_function = nn.SmoothL1Loss()
        l1loss_function = nn.L1Loss()

        optimizer = optim.Adam(model.parameters(), lr=1e-3)

        for epoch in range(1, max_epoches + 1):
            train_loss = 0
            for train_inputs, train_targets in train_dataloader:

                train_inputs  = train_inputs.float()
                train_targets = train_targets.float()
                train_inputs  = train_inputs.to(device)
                train_targets = train_targets.to(device)

                optimizer.zero_grad()

                train_scores = model(train_inputs)

                loss = loss_function(train_scores, train_targets)
                loss.backward()
                optimizer.step()

                train_loss += loss.item()
            train_loss /= len(train_dataloader)

            with torch.no_grad():
                valid_inputs, valid_targets = iter(valid_dataloader).next()

                valid_inputs  = valid_inputs.float()
                valid_targets = valid_targets.float()
                valid_inputs  = valid_inputs.to(device)
                valid_targets = valid_targets.to(device)

                valid_scores = model(valid_inputs)

                loss = loss_function(valid_scores, valid_targets)
                valid_loss = loss.item() / len(valid_dataloader)

                val_scores  = valid_scores.to('cpu').detach().numpy().astype(np.float32)
                val_targets = valid_targets.to('cpu').detach().numpy().astype(np.float32)
                throughput_scores = valid_scores[:, 0]
                loss_rate_scores  = valid_scores[:, 1]
                throughput_targets = valid_targets[:, 0]
                loss_rate_targets  = valid_targets[:, 1]

                throughput_loss, loss_rate_loss = (l1loss_function(throughput_scores, throughput_targets).item(), l1loss_function(loss_rate_scores, loss_rate_targets).item())

            if epoch < 10 or epoch % log_seq == 0:
                print(f"Epoch: [{epoch}/{max_epoches}] train/valid loss: {train_loss:.4f} / {valid_loss:.4f} throughput/loss rate: {throughput_loss:.4f} / {loss_rate_loss:.4f}")

            if epoch % 500 == 0:
                torch.save(model.state_dict(), f"{model_dir}/fold{i + 1}_{epoch}.mdl")

def test(input_path="/content/drive/MyDrive/similarity_measures/test/issue/similarity_0.json", model_path="/content/models/fold1_1000.mdl", seed=1):
    device = torch.device("cuda" if torch.cuda.is_available() else "cpu")

    similarity_path = input_path
    model_path     = model_path
    seed           = seed

    set_seed(seed)

    sequence_length = 1000
    hidden_dim      = 128
    num_layers      = 2
    max_epoches     = 1000

    test_dataset = SimilarityDataset(
            path=similarity_path,
            sequence_length=sequence_length,
            device=device,
            transform=None,
            is_train=False
            )

    test_dataloader = DataLoader(test_dataset, batch_size=len(test_dataset), shuffle=True, collate_fn=collate_fn)

    input_dim  = test_dataset.input_dim
    target_dim = test_dataset.target_dim

    model = NetworkStateEstimationFromVideoStreaming(input_dim, target_dim, sequence_length, hidden_dim).to(device)
    model.load_state_dict(torch.load(model_path))

    loss_function = nn.L1Loss()

    with torch.no_grad():
        test_inputs, test_targets = iter(test_dataloader).next()
        test_inputs  = test_inputs.float()
        test_targets = test_targets.float()

        test_scores = model(test_inputs)
        test_targets = test_targets.to(device)
        test_loss   = loss_function(test_scores, test_targets)

        throughput_scores = test_scores[:, 0]
        loss_rate_scores  = test_scores[:, 1]
        throughput_targets = test_targets[:, 0]
        loss_rate_targets  = test_targets[:, 1]

        throughput = throughput_scores.mean()
        loss_rate  = loss_rate_scores.mean()

        throughput_loss, loss_rate_loss = (loss_function(throughput_scores, throughput_targets).item(), loss_function(loss_rate_scores, loss_rate_targets).item())
        print(f"model: {model_path} throughput / loss rate {throughput} / {loss_rate} throughput loss/ loss rate loss: {throughput_loss:.4f} / {loss_rate_loss:.4f}")
train(log_seq=50)
test(input_path="/content/drive/MyDrive/similarity_measures/test/issue/similarity_0.json")<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.1017286632274414, 'fraction_numerical': 0.007783509819893203, 'mean_word_length': 2.689482470784641, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '1', 'dir': 'jupyter-structured-clean-dedup', 'id': '84747', 'license': 'MIT', 'n_tokens_mistral': 3574, 'n_tokens_neox': 3342, 'n_words': 642, 'path': 'jupyter/ai_based_networking_junior_challenge.ipynb', 'repo_name': 'oakeshott/ai-based-networking-2021', 'size': '11049'}