<jupyter_start><jupyter_text>Anomaly Detection   0 - Imports<jupyter_code>%%capture
import json
from pandas import concat, DataFrame
from sklearn.tree import DecisionTreeClassifier
from sklearn.model_selection import train_test_split
from sklearn.metrics import accuracy_score, f1_score, recall_score, precision_score

# Proprietary 
from ydata.connectors import LocalConnector
from ydata.connectors.filetype import FileType<jupyter_output><empty_output><jupyter_text>1 - Load Data<jupyter_code># Initialize connector and read data
connector = LocalConnector()

# Read the data. Augumented and original
data_processed = connector.read_file('data_processed.csv', file_type=FileType.CSV).to_pandas()
data_sampled = connector.read_file('data_sampled.csv', file_type=FileType.CSV).to_pandas()
x_test = connector.read_file('test_set.csv', file_type = FileType.CSV).to_pandas().set_index('test_ind')

# Concatenate synthesized data and original data
aug_df = concat([data_processed, data_sampled])<jupyter_output><empty_output><jupyter_text>2 - Classification   2.1 - Split the Data<jupyter_code># Split the data into train and test
x_train_augmented = aug_df.drop(x_test.index)<jupyter_output><empty_output><jupyter_text>2.2 - Model Training<jupyter_code># Create the DecisionTreeClassifier. One for the normal data and another for the augmented data.
aug_tree_clf = DecisionTreeClassifier(random_state=4)

# Fit the model and predict
aug_tree_clf.fit(x_train_augmented.drop('Unusual', axis=1), x_train_augmented['Unusual'])<jupyter_output><empty_output><jupyter_text>2.3 - Predict<jupyter_code>aug_preds = aug_tree_clf.predict(x_test.drop('Unusual', axis=1))<jupyter_output><empty_output><jupyter_text>2.4 - Evaluate Predictions<jupyter_code># Look at F1 score and Accuracy
aug_acc = accuracy_score(x_test['Unusual'].values, aug_preds)
aug_f1 = f1_score(x_test['Unusual'].values, aug_preds)
aug_recall = recall_score(x_test['Unusual'].values, aug_preds)
aug_precision = precision_score(x_test['Unusual'].values, aug_preds)<jupyter_output><empty_output><jupyter_text>3 - Create Artifacts<jupyter_code># Create Artifact. The table with the metrics will be shown on the "Run Output" section of the "Runs". 
metrics = {
    'metrics': [
         {
            'name': 'augmented-accuracy-score',
            'numberValue':  aug_acc,
            'format': 'PERCENTAGE'
        },
         {
            'name': 'augmented-f1-score',
            'numberValue':  aug_f1,
            'format': 'PERCENTAGE'
        },
        {
            'name': 'augmented-recall-score',
            'numberValue':  aug_recall,
            'format': 'PERCENTAGE'
        },
        {
            'name': 'augmented-precision-score',
            'numberValue':  aug_precision,
            'format': 'PERCENTAGE'
        },
    ]
  }

with open("mlpipeline-metrics.json", 'w') as f:
    json.dump(metrics, f)
from sklearn.metrics import confusion_matrix 

pos_neg = confusion_matrix(x_test['Unusual'].values, aug_preds).ravel()

matrix = [
    ['normal', 'normal', pos_neg[0]],
    ['normal', 'unusual', pos_neg[1]],
    ['unusual', 'normal', pos_neg[2]],
    ['unusual', 'unusual', pos_neg[3]]
]

df = DataFrame(matrix,columns=['target','predicted','count'])

metadata = {
    "outputs": [
        {
            "type": "confusion_matrix",
            "format": "csv",
            "schema": [
                {
                    "name": "target",
                    "type": "CATEGORY"
                },
                {
                    "name": "predicted",
                    "type": "CATEGORY"
                },
                {
                    "name": "count",
                    "type": "NUMBER"
                }
            ],
            "source": df.to_csv(header=False, index=False),
            "storage": "inline",
            "labels": [
                "normal",
                "unusual"
            ]
        }
    ]
}

with open('mlpipeline-ui-metadata.json', 'w') as f:
    json.dump(metadata, f)<jupyter_output><empty_output>
{'fraction_non_alphanumeric': 0.12648809523809523, 'fraction_numerical': 0.006200396825396825, 'mean_word_length': 3.037037037037037, 'pattern_counts': {'":': 13, '<': 29, '<?xml version=': 0, '>': 29, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'chain_length': '7', 'dir': 'jupyter-structured-clean-dedup', 'id': '625168', 'license': 'MIT', 'n_tokens_mistral': 1364, 'n_tokens_neox': 1302, 'n_words': 241, 'path': '5 - synthetic-data-applications/regular-tabular/mobile_network-anomaly_detection/7_train_and_detect_augmented.ipynb', 'repo_name': 'ydataai/Blog', 'size': '4032'}