<jupyter_start><jupyter_text>CNN model for 攻撃自動検知ワークショップ in SECCON 2019 https://connpass.com/event/159753/  seccon_ws_detection  auther : ＠m3yrin Model Specification* Char-Base CNN sequence modeling Memo* Bulit on AllenNLP* Tested on Google Colaboratory  Preparation<jupyter_code>!git clone https://github.com/palloc/seccon_ws_detection.git
!pip install allennlp
%cd seccon_ws_detection
!wget -P dataset https://raw.githubusercontent.com/palloc/seccon_ws_detection/score/dataset/level2_train.tsv
!ls
# filepath configs
import os

full_path = './'
DATA_DIR = os.path.join(full_path, 'dataset')

#ORIGINAL_FILE = 'level1_train.tsv'
ORIGINAL_FILE = 'level2_train.tsv'
ORIGINAL_PATH = os.path.join(DATA_DIR, ORIGINAL_FILE)

#TRAIN_FILE = 'level1_split_train.tsv'
TRAIN_FILE = 'level2_split_train.tsv'
TRAIN_PATH = os.path.join(DATA_DIR, TRAIN_FILE)

#VALID_FILE = 'level_split_valid.tsv'
VALID_FILE = 'level2_split_valid.tsv'
VALID_PATH = os.path.join(DATA_DIR, VALID_FILE)

AUG_FILE = 'train_aug.tsv'
AUG_PATH = os.path.join(DATA_DIR, AUG_FILE)

# for random seed.
random_state = 32<jupyter_output><empty_output><jupyter_text>EDA<jupyter_code>import codecs
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt

np.random.seed(seed=random_state)
# Load dataset.
def load_dataset(path, num_augment = None):
    dataset = []
    X = []
    y = []
    with codecs.open(path, mode='r', encoding='utf-8') as fin:
        dataset.extend(fin.readlines())
    
    for payload in dataset:
        X.append(payload.split('\t')[0])

        label = payload.split('\t')[1].replace('\n', '')
        label = int(label)
        y.append(label)

    # Data augmentation
    # sampling random 2 indices and concatinating them.
    if num_augment is not None:
        X_aug = []
        y_aug = []

        ids_list = np.random.randint(len(X), size=(num_augment, 2))

        for ids in ids_list:
            X_concat = X[ids[0]] + X[ids[1]]

            y_concat = 1 if y[ids[0]] + y[ids[1]] > 0 else 0

            X_aug.append(X_concat)
            y_aug.append(y_concat)
        
        # Also added original sequences
        return  X_aug + X, y_aug + y

    return X, y
X, y = load_dataset(ORIGINAL_PATH)
data_df = pd.DataFrame(zip(X, y))
data_df.columns = ['X','y']
data_df.head()
data_df.describe()
# check the distribution of y
data_df.y.value_counts().plot(kind="bar")
plt.show()<jupyter_output><empty_output><jupyter_text>File Split<jupyter_code>test_size = 0.2
from sklearn.model_selection import train_test_split
train_X, valid_X, train_y, valid_y = train_test_split(X, y, test_size=test_size, random_state=random_state)
train_df = pd.DataFrame(zip(train_X, train_y))
valid_df = pd.DataFrame(zip(valid_X, valid_y))
train_df.to_csv(TRAIN_PATH, sep='\t', index=False, header=False)
valid_df.to_csv(VALID_PATH, sep='\t', index=False, header=False)<jupyter_output><empty_output><jupyter_text>Data Augmentation<jupyter_code>num_augment = 400
X, y = load_dataset(TRAIN_PATH, num_augment = num_augment)

data_auged_df = pd.DataFrame(zip(X, y))
data_auged_df.columns = ['X','y']
data_auged_df.head()
data_auged_df.describe()
data_auged_df.y.value_counts().plot(kind="bar")
plt.show()
data_auged_df.to_csv(AUG_PATH, sep='\t' , index = False, header=False)
!ls dataset<jupyter_output>level1_test.tsv   level2_split_train.tsv  level2_train.tsv
level1_train.tsv  level2_split_valid.tsv  train_aug.tsv<jupyter_text>Modeling<jupyter_code># Parameters

## for model
char_min_count = 1
batch_size = 32
hidden_dim = 64
dropout_prob = 0.2

## for training
LR = 1e-4
PATIENCE = 50
EPOCHS = 100
import os
import codecs

import numpy as np
import pandas as pd
import logging

import itertools
import json
import logging
import string
from collections import defaultdict
from typing import Dict, List, Union, Tuple, Any, Optional

from overrides import overrides
from word2number.w2n import word_to_num

import torch

import allennlp
from allennlp.common.file_utils import cached_path
from allennlp.data.fields import (
    Field,
    TextField,
    MetadataField,
    LabelField,
    ListField,
    SequenceLabelField,
    SpanField,
    IndexField,
)

from allennlp.data.dataset_readers.dataset_reader import DatasetReader
from allennlp.data.instance import Instance
from allennlp.data.tokenizers import Token, Tokenizer
from allennlp.data.token_indexers.token_indexer import TokenIndexer
# logging setting
logging.basicConfig(format='%(message)s', level=logging.INFO)
logger = logging.getLogger(__name__)
class Lv1Reader(DatasetReader):

    def __init__(
        self,
        tokenizer: Tokenizer = None,
        token_indexers: Dict[str, TokenIndexer] = None,
        lazy: bool = False,
    ) -> None:
        super().__init__(lazy)
        self._tokenizer = tokenizer or WhitespaceTokenizer()
        self._token_indexers = token_indexers

    @overrides
    def _read(self, file_path: str):
        file_path = cached_path(file_path)
        
        with codecs.open(file_path, mode='r', encoding='utf-8') as dataset_file:
            for line in dataset_file:
                
                example = line.split('\t')
                payload = example[0]
                payload = self._tokenizer.tokenize(payload)

                target = None
                if len(example) == 2:
                    target = example[1].replace('\n', '')
                    target = int(target)
                
                instance = self.text_to_instance(
                    payload,
                    target,
                )
                yield instance
                

    @overrides
    def text_to_instance(
        self,  # type: ignore
        payload: str,
        target: int,
        ) -> Union[Instance, None]:
        
        fields: Dict[str, Field] = {}
        fields["payload"] = TextField(payload, self._token_indexers)

        if target is not None:
            fields["target"] = LabelField(target, skip_indexing=True)
        
        return Instance(fields)
from allennlp.data.tokenizers.character_tokenizer import CharacterTokenizer
from allennlp.data.token_indexers.token_characters_indexer import TokenCharactersIndexer
from allennlp.data.token_indexers.single_id_token_indexer import SingleIdTokenIndexer


reader = Lv1Reader(tokenizer = CharacterTokenizer(),
                   token_indexers = {"tokens": SingleIdTokenIndexer(lowercase_tokens = True)}
                  )

train_dataset = reader.read(AUG_PATH)
valid_dataset = reader.read(VALID_PATH)
from allennlp.data.vocabulary import Vocabulary
vocab = Vocabulary.from_instances(train_dataset, min_count={'tokens': char_min_count})
vocab
from allennlp.data.iterators import BasicIterator, BucketIterator

iterator = BucketIterator(
    batch_size=batch_size,
    sorting_keys=[("payload", "num_tokens")],
)
iterator.index_with(vocab)
from allennlp.modules.token_embedders import Embedding
from allennlp.modules.text_field_embedders import BasicTextFieldEmbedder

embedding = Embedding(num_embeddings=vocab.get_vocab_size(), embedding_dim=hidden_dim, padding_index=0)
embedder = BasicTextFieldEmbedder({"tokens": embedding})
from allennlp.nn.util import get_text_field_mask

from allennlp.models.model import Model
from allennlp.modules import FeedForward, Seq2SeqEncoder, Seq2VecEncoder, TextFieldEmbedder
from allennlp.nn import InitializerApplicator, RegularizerApplicator
from allennlp.training.metrics import CategoricalAccuracy
from allennlp.training.metrics.auc import Auc

class Lv1Net(Model):

    def __init__(
        self,
        vocab: Vocabulary,
        text_field_embedder: TextFieldEmbedder,
        seq2vec_encoder: Seq2VecEncoder,
        num_labels: int,
        feedforward: Optional[FeedForward] = None,
        dropout_prob: float = 0.1,
        initializer: InitializerApplicator = InitializerApplicator(),
        regularizer: Optional[RegularizerApplicator] = None,
    ) -> None:
        super().__init__(vocab, regularizer)

        self._text_field_embedder = text_field_embedder
        text_embed_dim = text_field_embedder.get_output_dim()
        self._dropout = torch.nn.Dropout(p=dropout_prob)

        self._seq2vec_encoder = seq2vec_encoder
        self._feedforward = feedforward
        self._num_labels = num_labels
        
        if feedforward is not None:
            self._classifier_input_dim = self._feedforward.get_output_dim()
        else:
            self._classifier_input_dim = self._seq2vec_encoder.get_output_dim()

        self._classification_layer = torch.nn.Linear(self._classifier_input_dim, self._num_labels)

        self._accuracy = CategoricalAccuracy()
        self._auc = Auc(positive_label = 1)
        self._loss = torch.nn.CrossEntropyLoss()
        initializer(self)
        

    def forward(  # type: ignore
        self,
        payload: Dict[str, torch.LongTensor],
        target: torch.IntTensor = None,
    ) -> Dict[str, torch.Tensor]:

        
        mask = get_text_field_mask(payload).float()


        # Shape: (batch_size, payload_length, hidden)
        embedded_text = self._text_field_embedder(payload)
        embedded_text = self._seq2vec_encoder(embedded_text, mask=mask)

        embedded_text = self._dropout(embedded_text)

        if self._feedforward is not None:
            embedded_text = self._feedforward(embedded_text)

        logits = self._classification_layer(embedded_text)
        probs = torch.nn.functional.softmax(logits, dim=-1)

        output_dict = {"logits": logits, "probs": probs}

        if target is not None:
            loss = self._loss(logits, target.long().view(-1))
            output_dict["loss"] = loss
            self._accuracy(logits, target)
            self._auc(logits.argmax(dim=1), target)
        
        return output_dict

    @overrides
    def decode(self, output_dict: Dict[str, torch.Tensor]) -> Dict[str, torch.Tensor]:
        """
        Does a simple argmax over the probabilities, converts index to string label, and
        add ``"label"`` key to the dictionary with the result.
        """
        predictions = output_dict["probs"]
        if predictions.dim() == 2:
            predictions_list = [predictions[i] for i in range(predictions.shape[0])]
        else:
            predictions_list = [predictions]
        classes = []
        for prediction in predictions_list:
            label_idx = prediction.argmax(dim=-1).item()
            label_str = str(label_idx)
            classes.append(label_str)
        output_dict["label"] = classes
        return output_dict

    def get_metrics(self, reset: bool = False) -> Dict[str, float]:
        metrics = {"accuracy": self._accuracy.get_metric(reset), "auc": self._auc.get_metric(reset)}
        return metrics
from allennlp.modules.seq2vec_encoders.cnn_encoder import CnnEncoder
encoder = CnnEncoder(embedding_dim = hidden_dim, num_filters = hidden_dim)
model = Lv1Net(vocab = vocab,
               text_field_embedder = embedder,
               seq2vec_encoder = encoder,
               num_labels = 2,
               dropout_prob = dropout_prob
               )
import torch.optim as optim

optimizer = optim.Adam(model.parameters(), lr=LR)
if torch.cuda.is_available():
    print("cuda")
    cuda_device = 0
    model = model.cuda(cuda_device)
else:
    print("cpu")
    cuda_device = -1
from allennlp.training.trainer import Trainer

trainer = Trainer(model = model,
                  optimizer = optimizer,
                  iterator = iterator,
                  train_dataset = train_dataset,
                  validation_dataset = valid_dataset,
                  patience = PATIENCE,
                  validation_metric = "+auc",
                  #validation_metric = "+accuracy",
                  num_epochs = EPOCHS,
                  cuda_device = cuda_device
                 )
trainer.train()
# Model Save
with open("./model.th", 'wb') as f:
    torch.save(model.state_dict(), f)
vocab.save_to_files("./vocabulary")<jupyter_output><empty_output><jupyter_text>Prediction<jupyter_code># Model Reload
vocab2 = Vocabulary.from_files("./vocabulary")
model2 = Lv1Net(vocab = vocab,
               text_field_embedder = embedder,
               seq2vec_encoder = encoder,
               num_labels = 2,
               dropout_prob = dropout_prob
               )

with open("./model.th", 'rb') as f:
    model2.load_state_dict(torch.load(f))
if cuda_device > -1:
    model2.cuda(cuda_device)
from allennlp.predictors.predictor import Predictor

predictor = Predictor(model2, reader)
level1_pred = []

from tqdm import tqdm

TEST_FILE = 'level1_test.tsv'
#TEST_FILE = 'level2_test.tsv'
TEST_PATH = os.path.join(DATA_DIR, TEST_FILE)
test_dataset = reader.read(TEST_PATH)

for instance in tqdm(test_dataset):
    ans = predictor.predict_instance(instance)
    level1_pred.append(ans['label'])
level1_pred_df = pd.DataFrame(level1_pred)
level1_pred_df.T
level1_pred_df.T.to_csv('level1_pred.csv', index = False, header=False)
from google.colab import files
files.download('level1_pred.csv')<jupyter_output><empty_output><jupyter_text>Evaluation (Only on Google Colab)If you have labeled test data distributed by administrators, you can calculate performance score.  Please put them into your google drive and mount it by commands below.<jupyter_code>from google.colab import drive
drive.mount('/content/drive')
LABEL_PATH = "/content/drive/My Drive/SECCON/level2_test.tsv" # Path to labeled data
label_dataset = reader.read(LABEL_PATH)
# Model Reload
vocab2 = Vocabulary.from_files("./vocabulary")
model2 = Lv1Net(vocab = vocab,
               text_field_embedder = embedder,
               seq2vec_encoder = encoder,
               num_labels = 2,
               dropout_prob = dropout_prob
               )

with open("./model.th", 'rb') as f:
    model2.load_state_dict(torch.load(f))
if cuda_device > -1:
    model2.cuda(cuda_device)
from allennlp.training.util import evaluate
metrics = evaluate(model, label_dataset, iterator, cuda_device, batch_weight_key="")
metrics<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.09850107066381156, 'fraction_numerical': 0.008279800142755174, 'mean_word_length': 3.230374396135266, 'pattern_counts': {'":': 6, '<': 28, '<?xml version=': 0, '>': 37, 'https://': 3, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 1, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '281628', 'license': 'MIT', 'n_tokens_mistral': 4637, 'n_tokens_neox': 4524, 'n_words': 928, 'path': 'seccon_ws_detection_m3yrin.ipynb', 'repo_name': 'm3yrin/seccon2019_ws_detection', 'size': '14010'}