<commit_before># -*- coding: utf-8 -*-

import cPickle as pickle
import itertools
import logging
import os

from sklearn import linear_model

from ..corpus import sensem
from ..experiment import results, supervised
from ..model import autoencoder
from ..processor import bowprocessor, vecprocessor
from ..utils.setup_logging import setup_logging

setup_logging()
logger = logging.getLogger(__name__)


class SupervisedPipeline(object):
    processors_map = {
        'bow': bowprocessor.BoWProcessor,
        'bopos': bowprocessor.BoPoSProcessor,
        'pos': bowprocessor.PoSProcessor,
        'wordvec': vecprocessor.WordVectorsProcessor
    }

    models_map = {
        'logreg': linear_model.LogisticRegression,
        'autoencoder': autoencoder.StackedDenoisingAutoencoder
    }

    def __init__(self, corpus_directory, results_directory, **kwargs):
        self._corpus_iterator = sensem.SenSemCorpusDirectoryIterator(
            corpus_directory, kwargs.pop('binary_corpus', False)
        )
        self._results_directory = results_directory
        self._iterations = kwargs.pop('iterations', 5)
        self._processors = kwargs.pop('processors', [])
        # List of tuples. For each tuple the first argument is a keyword in processors_map, the second argument
        # is a dictionary with the processors parameters
        self._models = kwargs.pop('models', [])
        # Similar to _processors, but each tuple has a model and its parameters
        self._save_corpus_path = kwargs.pop('save_corpus_path', u"")
        self._save_datasets_path = kwargs.pop('save_datasets_path', u"")
        self._load_datasets_path = kwargs.pop('load_datasets_path', u"")

    def _run_for_corpus(self, corpus):
        """
        :param corpus: ddnwsd.corpus.sensem.SenSemCorpus
        """

        experiments_dir = os.path.join(self._results_directory, corpus.lemma)

        for (pkey, pparam), (mkey, mparam) in itertools.product(self._processors, self._models):
            experiment_name = u"{}_{}".format(pkey, mkey)
            results_save_path = os.path.join(experiments_dir, experiment_name)
            os.makedirs(results_save_path)

            processor = self.processors_map[pkey](corpus, **pparam)
            """:type : dnnwsd.processor.base.BaseProcessor"""
            dataset_path = os.path.join(self._load_datasets_path, u"{}_{}.npz".format(corpus.lemma, pkey))

            if os.path.isfile(dataset_path):
                processor.load_data(dataset_path)
            else:
                processor.instances()

            if self._save_datasets_path:
                dataset_path = os.path.join(self._save_datasets_path, u"{}_{}.npz".format(corpus.lemma, pkey))
                if not os.path.isfile(dataset_path):  # Check if wasn't already created
                    processor.save_data(dataset_path)

            if mkey == 'autoencoder':
                mparam['classes_amount'] = len(processor.labels)
                mparam['layers'].insert(0, processor.dataset.shape[1])

            model = self.models_map[mkey](**mparam)
            """:type : dnnwsd.models.base.BaseModel"""

            logger.info(u"Running experiments for {} and model {}".format(processor.name, model.__class__.__name__))
<commit_msg>Change the experiment set iterator in supervised pipeline<commit_after># -*- coding: utf-8 -*-

import cPickle as pickle
import logging
import os

from sklearn import linear_model

from ..corpus import sensem
from ..experiment import results, supervised
from ..model import autoencoder
from ..processor import bowprocessor, vecprocessor
from ..utils.setup_logging import setup_logging

setup_logging()
logger = logging.getLogger(__name__)


class SupervisedPipeline(object):
    processors_map = {
        'bow': bowprocessor.BoWProcessor,
        'bopos': bowprocessor.BoPoSProcessor,
        'pos': bowprocessor.PoSProcessor,
        'wordvec': vecprocessor.WordVectorsProcessor
    }

    models_map = {
        'logreg': linear_model.LogisticRegression,
        'autoencoder': autoencoder.StackedDenoisingAutoencoder
    }

    def __init__(self, corpus_directory, results_directory, **kwargs):
        self._corpus_iterator = sensem.SenSemCorpusDirectoryIterator(
            corpus_directory, kwargs.pop('binary_corpus', False), kwargs.pop('sense_filter', 3)
        )
        self._results_directory = results_directory
        self._iterations = kwargs.pop('iterations', 5)
        self._experiment_set = kwargs.pop('experiment_set', [])
        # List of 4-tuples, each defining an experiment.
        # (processor, processor_parameters, model, model_parameters)
        self._save_corpus_path = kwargs.pop('save_corpus_path', u"")
        self._save_datasets_path = kwargs.pop('save_datasets_path', u"")
        self._load_datasets_path = kwargs.pop('load_datasets_path', u"")

    def _run_for_corpus(self, corpus):
        """
        :param corpus: ddnwsd.corpus.sensem.SenSemCorpus
        """

        experiments_dir = os.path.join(self._results_directory, corpus.lemma)

        for (pkey, pparam, mkey, mparam) in self._experiment_set:
            experiment_name = u"{}_{}".format(pkey, mkey)
            results_save_path = os.path.join(experiments_dir, experiment_name)
            os.makedirs(results_save_path)

            processor = self.processors_map[pkey](corpus, **pparam)
            """:type : dnnwsd.processor.base.BaseProcessor"""
            dataset_path = os.path.join(self._load_datasets_path, u"{}_{}.npz".format(corpus.lemma, pkey))

            if os.path.isfile(dataset_path):
                processor.load_data(dataset_path)
            else:
                processor.instances()

            if self._save_datasets_path:
                dataset_path = os.path.join(self._save_datasets_path, u"{}_{}.npz".format(corpus.lemma, pkey))
                if not os.path.isfile(dataset_path):  # Check if wasn't already created
                    processor.save_data(dataset_path)

            if mkey == 'autoencoder':
                mparam['classes_amount'] = len(processor.labels)
                mparam['layers'].insert(0, processor.dataset.shape[1])

            model = self.models_map[mkey](**mparam)
            """:type : dnnwsd.models.base.BaseModel"""

            logger.info(u"Running experiments for {} and model {}".format(processor.name, model.__class__.__name__))

{'fraction_non_alphanumeric': 0.10978670012547051, 'fraction_numerical': 0.0015683814303638645, 'mean_word_length': 2.929143561306223, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '217251', 'n_tokens_mistral': 1900, 'n_tokens_neox': 1853, 'n_words': 365}