<commit_before>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse

import numpy as np
import pandas as pd

from spam.common import DATASET_META
from spam.common.utils import get_file_path_list, split_dataset
from spam.preprocess import preprocess
from spam.deeplearning import StackedDenoisingAutoEncoder


UNLABEL = -1
HAM = 0
SPAM = 1

parser = argparse.ArgumentParser()
parser.add_argument('-c', '--csv', action='store_true')
parser.add_argument('-n', '--npz', action='store_true')
parser.add_argument('-m', '--model', action='store_true')
args = parser.parse_args()

if args.csv:
    file_path_list = get_file_path_list(DATASET_META)

    print('Spliting the dataset..')
    unlabeled_path, (train_path, train_class), \
        (test_path, test_class) = split_dataset(file_path_list, seed=1337)

    # generate panda dataframes and export it to csv
    print('Generating unlabeled dataframe..')
    unlabeled_data = pd.DataFrame(
        data={
            'email': [preprocess.read_email(path) for path in unlabeled_path],
            'class': [UNLABEL for _ in range(len(unlabeled_path))]
        },
        columns=['email', 'class'],
    )

    print('Generating train dataframe..')
    train_data = pd.DataFrame(
        data={
            'email': [preprocess.read_email(path) for path in train_path],
            'class': [SPAM if cl == 'spam' else HAM for cl in train_class]
        },
        columns=['email', 'class'],
    )

    print('Generating test dataframe..')
    test_data = pd.DataFrame(
        data={
            'email': [preprocess.read_email(path) for path in test_path],
            'class': [SPAM if cl == 'spam' else HAM for cl in test_class]
        },
        columns=['email', 'class'],
    )

    print('Exporting dataframes into a csv files inside data/csv/ ..')
    unlabeled_data.to_csv('unlabeled_data.csv')
    train_data.to_csv('train_data.csv')
    test_data.to_csv('test_data.csv')

if args.npz:
    print('Reading csv files..')
    unlabeled_data = pd.read_csv('data/csv/unlabeled_data.csv',
                                 encoding='iso-8859-1')
    train_data = pd.read_csv('data/csv/train_data.csv',
                             encoding='iso-8859-1')
    test_data = pd.read_csv('data/csv/test_data.csv',
                            encoding='iso-8859-1')

    print('Generating feature vectors..')
    unlabeled_feat, _ = preprocess.count_vectorizer([unlabeled_data['email']])
    train_feat, test_feat = preprocess.count_vectorizer([
        train_data['email'], test_data['email']
    ])

    print('Exporting npz files inside data/npz/ ..')
    np.savez('data/npz/unlabeled_feature', X=unlabeled_feat)
    np.savez('data/npz/train_feature',
             X=train_feat, Y=train_data['class'].values)
    np.savez('data/npz/test_feature',
             X=test_feat, Y=test_data['class'].values)

if args.model:
    print('Building model..')
    sda = StackedDenoisingAutoEncoder(
        batch_size=128, classes=2, epochs=0, n_folds=4,
        hidden_layers=[2500, 1700, 1000, 300, ],
        noise_layers=[0.3, 0.2, 0.1, ],
    )
    model = sda.build_sda()

    model.add(sda.build_finetune())
    X_train, Y_train = sda.dataset['train_data']
    X_test, Y_test = sda.dataset['train_data']

    print('Finetuning the model..')
    model.fit(
        X_train, Y_train, batch_size=sda.batch_size,
        nb_epoch=sda.epochs, show_accuracy=True,
        validation_data=(X_test, Y_test), validation_split=0.1,
    )

    print('Evaluating model..')
    score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)

    print('Test score: {}'.format(score[0]))
    print('Test accuracy: {}'.format(score[1]))

    print('Saving model structure and weights..')
    open('spam_ms_epochs_{}.json'.format(sda.epochs), 'w') \
        .write(model.to_json())
    model.save_weights('spam_mw_epochs_{}.hdf5'<commit_msg>Add --all args and fix logs and fix saving paths.<commit_after>#!/usr/bin/env python
# -*- coding: utf-8 -*-

import argparse

import numpy as np
import pandas as pd

from keras.optimizers import SGD
from spam.common import DATASET_META
from spam.common.utils import get_file_path_list, split_dataset
from spam.preprocess import preprocess
from spam.deeplearning import StackedDenoisingAutoEncoder


UNLABEL = -1
HAM = 0
SPAM = 1

parser = argparse.ArgumentParser()
parser.add_argument('-a', '--all', action='store_true')
parser.add_argument('-c', '--csv', action='store_true')
parser.add_argument('-n', '--npz', action='store_true')
parser.add_argument('-m', '--model', action='store_true')
args = parser.parse_args()

if args.csv or args.all:
    file_path_list = get_file_path_list(DATASET_META)

    print('Spliting the dataset..')
    unlabeled_path, (train_path, train_class), \
        (test_path, test_class) = split_dataset(file_path_list, seed=1337)

    # generate panda dataframes and export it to csv
    print('Generating unlabeled dataframe..')
    unlabeled_data = pd.DataFrame(
        data={
            'email': [preprocess.read_email(path) for path in unlabeled_path],
            'class': [UNLABEL for _ in range(len(unlabeled_path))]
        },
        columns=['email', 'class'],
    )

    print('\nGenerating train dataframe..')
    train_data = pd.DataFrame(
        data={
            'email': [preprocess.read_email(path) for path in train_path],
            'class': [SPAM if cl == 'spam' else HAM for cl in train_class]
        },
        columns=['email', 'class'],
    )

    print('\nGenerating test dataframe..')
    test_data = pd.DataFrame(
        data={
            'email': [preprocess.read_email(path) for path in test_path],
            'class': [SPAM if cl == 'spam' else HAM for cl in test_class]
        },
        columns=['email', 'class'],
    )

    print('\nExporting dataframes into a csv files inside data/csv/ ..')
    unlabeled_data.to_csv('data/csv/unlabeled_data.csv')
    train_data.to_csv('data/csv/train_data.csv')
    test_data.to_csv('data/csv/test_data.csv')

if args.npz or args.all:
    print('Reading csv files..')
    unlabeled_data = pd.read_csv('data/csv/unlabeled_data.csv',
                                 encoding='iso-8859-1')
    train_data = pd.read_csv('data/csv/train_data.csv',
                             encoding='iso-8859-1')
    test_data = pd.read_csv('data/csv/test_data.csv',
                            encoding='iso-8859-1')

    print('Generating feature vectors..')
    unlabeled_feat, _ = preprocess.count_vectorizer([unlabeled_data['email']])
    train_feat, test_feat = preprocess.count_vectorizer([
        train_data['email'], test_data['email']
    ])

    print('Exporting npz files inside data/npz/ ..')
    np.savez('data/npz/unlabeled_feature', X=unlabeled_feat)
    np.savez('data/npz/train_feature',
             X=train_feat, Y=train_data['class'].values)
    np.savez('data/npz/test_feature',
             X=test_feat, Y=test_data['class'].values)

if args.model or args.all:
    print('Building model..')
    sda = StackedDenoisingAutoEncoder(
        batch_size=128, classes=2, epochs=0, n_folds=4,
        hidden_layers=[5000, 3500, 2000, 500, ],
        noise_layers=[0.3, 0.2, 0.1, ],
    )
    model = sda.build_sda()

    model.add(sda.build_finetune())

    sgd = SGD()
    model.compile(loss='categorical_crossentropy', optimizer=sgd)

    X_train, Y_train = sda.dataset['train_data']
    X_test, Y_test = sda.dataset['train_data']

    print('Finetuning the model..')
    model.fit(
        X_train, Y_train, batch_size=sda.batch_size,
        nb_epoch=sda.epochs, show_accuracy=True,
        validation_data=(X_test, Y_test), validation_split=0.1,
    )

    print('Evaluating model..')
    score = model.evaluate(X_test, Y_test, show_accuracy=True, verbose=0)

    print('Test score: {}'.format(score[0]))
    print('Test accuracy: {}'.format(score[1]))

    print('Saving model structure and weights..')
    open('spam_ms_epochs_{}.json'.format(sda.epochs), 'w') \
        .write(model.to_json())
    model.save_weights('spam_mw_epochs_{}.hdf5'
{'fraction_non_alphanumeric': 0.14225317085302164, 'fraction_numerical': 0.013926883859736384, 'mean_word_length': 3.088967971530249, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '149163', 'n_tokens_mistral': 2832, 'n_tokens_neox': 2704, 'n_words': 523}