<commit_before>"""
Preliminary code for submissions on the
Microsoft Malware Classification challenge.
"""

__authors__ = '<NAME>, <NAME>'
__licence__ = 'Apache'
__email__ = '<EMAIL>'

import sys, os, argparse
import pymongo
import numpy as np
from sklearn import linear_model
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.decomposition import TruncatedSVD
import joblib

def _init_connection(server_addr='172.16.17.32'):
    """
    Creates a connection with the mongo database using the
    default parameters and returns the malware database.
    If want to modify, should do it from the code.
    """
    if server_addr == 'andres':
        server_addr = 'afruizc-office.cs.unm.edu'
    client = pymongo.MongoClient(server_addr)
    client.malware.authenticate("query", "malware_challenge")
    return client.malware

class Executor(object):
    """
    Executes the selected classification pipeline. Right now the
    custmization process is by hand, i.e. you have to code it. The
    idea is to have a couple of pipelines prepared
    """

    def __init__(self):
        """
        Creates a new executor object and initializes the main
        components.
        """
        self.target_names = ['Ramnit', 'Lollipop', 'Kelihos_ver3',
                             'Vundo', 'Simda', 'Tracur', 'Kelihos_ver1',
                             'Obfuscator.ACY', 'Gatak']
        self.db = _init_connection(server_addr='andres')
        self.train = None
        self.test = None
        self.param_tunning = None
        self.fitted_model = None

    def _load_train(self):
        """
        Loads the training dataset.
        __THIS__ is the method you want to modify when querying
        the database.
        TODO: The data part can be just one function
        """
        data_train = [('\n'.join(x['ida_comments']), x['class'])
                          for x in self.db.samples.find({
                          "id": {"$exists": True}, # The id field exists
                          "ida_comments ": {"$ne": ""}})]
                            # ^ For non-empty field
        return list(zip(*data_train))

    def _load_test(self):
        """
        Loads the testing dataset.
        __THIS__ is the method you want to modify when querying
        the database.
        """
        data_test = [('\n'.join(x['ida_comments']), '"%s"' % x['id'])
                      for x in self.db.test_samples.find({
                            "id":{"$exists": True}, # The id field exists
                            "ida_comments": {"$ne": ""}})]
        return list(zip(*data_test))

    def load_data(self, training=True, testing=False):
        """
        Fetches the training data from the database. `training` and
        testing indicate the datasets that should be loaded.

        Arguments:
        `training`: If False, the training dataset is NOT loaded.
        `testng`: If True, the testing dataset IS loaded
        """
        if training:
            temp = self._load_train()
            self.train = {'data': temp[0], 'target': temp[1]}
        if testing:
            temp = self._load_test()
            self.test = {'data': temp[0], 'names':temp[1]}

    def config_model(self):
        """
        Configures the pipeline
        """
        pip = Pipeline([
                ('vectorizer', CountVectorizer(max_df=.7,
                               ngram_range=(1, 2))),
                ('freq_norm', TfidfTransformer()),
                ('lsa', TruncatedSVD(n_components=150)),
                ('classifier', linear_model.SGDClassifier(loss='log',
                                    penalty='l2', alpha=1e-2, n_jobs=-1))
        ])
        parameters = {}
        self.param_tunning = GridSearchCV(pip, parameters, n_jobs=-1)

    def fit(self):
        """
        Fits the parameters to the pipeline
        """
        self.fitted_model = self.param_tunning.fit(self.train['data'],
                                                   self.train['target'])

    def _predict(self, X, create_submission=False, filename='submission.txt'):
        """
        Predicts a set of 9 probabilities per malware sample, that
        correspond to the 9 malware classes. If `create_submission`<commit_msg>Reverted converting data to numpy arrays.<commit_after>"""
Preliminary code for submissions on the
Microsoft Malware Classification challenge.
"""

__authors__ = '<NAME>, <NAME>'
__licence__ = 'Apache'
__email__ = '<EMAIL>'

import sys, os, argparse
import pymongo
import numpy as np
from sklearn import linear_model
from sklearn.feature_extraction.text import CountVectorizer
from sklearn.feature_extraction.text import TfidfTransformer
from sklearn.pipeline import Pipeline
from sklearn.grid_search import GridSearchCV
from sklearn.decomposition import TruncatedSVD
from microsoft_malware_challenge.src.utils import utils
import joblib


class Executor(object):
    """
    Executes the selected classification pipeline. Right now the
    custmization process is by hand, i.e. you have to code it. The
    idea is to have a couple of pipelines prepared
    """

    def __init__(self):
        """
        Creates a new executor object and initializes the main
        components.
        """
        self.target_names = ['Ramnit', 'Lollipop', 'Kelihos_ver3',
                             'Vundo', 'Simda', 'Tracur', 'Kelihos_ver1',
                             'Obfuscator.ACY', 'Gatak']
        self.db = utils.get_mongodb()
        self.train = None
        self.test = None
        self.param_tunning = None
        self.fitted_model = None

    def _load_train(self):
        """
        Loads the training dataset.
        __THIS__ is the method you want to modify when querying
        the database.
        TODO: The data part can be just one function
        """
        data_train = [('\n'.join(x['ida_comments']), x['class'])
                          for x in self.db.samples.find({
                          "id": {"$exists": True}, # The id field exists
                          "ida_comments ": {"$ne": ""}})]
                            # ^ For non-empty field
        return list(zip(*data_train))

    def _load_test(self):
        """
        Loads the testing dataset.
        __THIS__ is the method you want to modify when querying
        the database.
        """
        data_test = [('\n'.join(x['ida_comments']), '"%s"' % x['id'])
                      for x in self.db.test_samples.find({
                            "id":{"$exists": True}, # The id field exists
                            "ida_comments": {"$ne": ""}})]
        return list(zip(*data_test))

    def load_data(self, training=True, testing=False):
        """
        Fetches the training data from the database. `training` and
        testing indicate the datasets that should be loaded.

        Arguments:
        `training`: If False, the training dataset is NOT loaded.
        `testng`: If True, the testing dataset IS loaded
        """
        if training:
            temp = self._load_train()
            self.train = {'data': (temp[0]), 'target': temp[1]}
        if testing:
            temp = self._load_test()
            self.test = {'data': (temp[0]), 'names': temp[1]}

    def config_model(self):
        """
        Configures the pipeline
        """
        pip = Pipeline([
                ('vectorizer', CountVectorizer(max_df=.8,
                               ngram_range=(1, 2))),
                ('freq_norm', TfidfTransformer()),
                ('classifier', linear_model.SGDClassifier(
                                    loss='modified_huber',
                                    penalty='elasticnet',
                                    alpha=1e-2,
                                    n_jobs=-1))
        ])
        parameters = {}
        self.param_tunning = GridSearchCV(pip, parameters, n_jobs=-1)

    def fit(self):
        """
        Fits the parameters to the pipeline
        """
        self.fitted_model = self.param_tunning.fit(self.train['data'],
                                                   self.train['target'])

    def _predict(self, X, create_submission=False, filename='submission.txt'):
        """
        Predicts a set of 9 probabilities per malware sample, that
        correspond to the 9 malware classes. If `create_submission`
{'fraction_non_alphanumeric': 0.10697455230914232, 'fraction_numerical': 0.005065975494816211, 'mean_word_length': 1.9588706866504009, 'pattern_counts': {'":': 16, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 11, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2249941', 'n_tokens_mistral': 2369, 'n_tokens_neox': 2295, 'n_words': 679}