<commit_before>import os
import json
import logging
import numpy as np
import eight_mile
import baseline
from copy import deepcopy
from baseline.reporting import create_reporting
from eight_mile.utils import (
    export,
    revlut,
    Offsets,
    get_env_gpus,
    import_user_module,
    listify,
)
from baseline.utils import (
    show_examples,
    normalize_backend,
)
from mead.downloader import EmbeddingDownloader, DataDownloader
from mead.utils import (
    index_by_label,
    get_mead_settings,
    print_dataset_info,
    read_config_file_or_json,
    get_dataset_from_key
)

from baseline.train import calc_lr_params

__all__ = []
exporter = export(__all__)
logger = logging.getLogger('mead')


def merge_reporting_with_settings(reporting, settings):
    default_reporting = settings.get('reporting_hooks', {})
    # Add default reporting information to the reporting settings.
    for report_type in default_reporting:
        if report_type in reporting:
            for report_arg, report_val in default_reporting[report_type].items():
                if report_arg not in reporting[report_type]:
                    reporting[report_type][report_arg] = report_val
    reporting_hooks = list(reporting.keys())
    for settings in reporting.values():
        for module in listify(settings.get('module', settings.get('modules', []))):
            import_user_module(module)
    return reporting_hooks, reporting



class Backend(object):
    """Simple object to represent a deep-learning framework backend"""
    def __init__(self, name=None, params=None, exporter=None):
        """Initialize the backend, optional with constructor args

        :param name: (``str``) Name of the framework: currently one of (`tensorflow`, `pytorch`)
        :param params: (``dict``) A dictionary of framework-specific user-data to pass through keyword args to each sub-module
        :param exporter: A framework-specific exporter to facilitate exporting to runtime deployment
        """
        self.name = normalize_backend(name)
        self.params = params

    def load(self, task_name=None):
        base_pkg_name = 'baseline.{}'.format(self.name)
        mod = import_user_module(base_pkg_name)
        import_user_module('eight_mile.{}.embeddings'.format(self.name))
        import_user_module('mead.{}.exporters'.format(self.name))
        if task_name is not None:
            import_user_module('{}.{}'.format(base_pkg_name, task_name))
        self.transition_mask = mod.transition_mask


TASK_REGISTRY = {}
@exporter
def register_task(cls):
    TASK_REGISTRY[cls.task_name()] = cls
    return cls

@exporter
def get_task_registry():
    return TASK_REGISTRY

@exporter
class Task(object):
    """Basic building block for a task of NLP problems, e.g. `tagger`, `classify`, etc.
    """

    def _create_backend(self, **kwargs):
        """This method creates and returns a `Backend` object

        :return:
        """
        pass

    def __init__(self, mead_settings_config=None):
        super(Task, self).__init__()
        self.config_params = None
        self.mead_settings_config = get_mead_settings(mead_settings_config)
        if 'datacache' not in self.mead_settings_config:
            self.data_download_cache = os.path.expanduser("~/.bl-data")
            self.mead_settings_config['datacache'] = self.data_download_cache
        else:
            self.data_download_cache = os.path.expanduser(self.mead_settings_config['datacache'])
        logger.info("using %s as data/embeddings cache", self.data_download_cache)

    @classmethod
    def task_name(cls):
        """This classmethod returns the official name of this task, e.g., `classify` for classification

        :return: (``str``) - String name of this task
        """
        pass

    def _create_vectorizers(self):
        """Read the `features` section of the mead config.  This sections contains both embedding info and vectorizers
        Then use the vectorizer sub-section to instantiate the vectorizers and return them in a ``dict`` with name
        keyed off of the `features->name` and value of `vectorizer`

        :return: (``dict``) - A dictionary of the vectorizers keyed by feature name
        """
        self.vectorizers = {}

        features = self.config_params['features']
        self.primary_key = features[0]['name']
        for feature in self.config_params['features']:
            key = feature['name']
            if '-' in key:
                raise ValueError('Feature names cannot contain "-". Found feature named "{}"'.format(key))
            if feature.get('primary', False) is True:
                self.primary_key = key
            vectorizer_section = feature.get('vectorizer', {'type': 'token1d'})
            vectorizer_section['mxlen'] = vectorizer_section.get('mxlen', self.config_params.get('preproc', {}).get('mxlen', -1))
            vectorizer_section['mxwlen'] = vectorizer_section.get('mxwlen', self.config_params.get('preproc', {}).get('mxwlen', -1))
            if 'transform' in vectorizer_section:
                vectorizer_section['transform_fn'] = eval(vectorizer_section['transform'])
            vectorizer = baseline.create_vectorizer(**vectorizer_section)
            self.vectorizers[key] = vectorizer


    @staticmethod
    def get_task_specific(task, mead_config):
        """Get the task from the task registry associated with the name

        :param task: The task name
        :return:
        """
        config = TASK_REGISTRY[task](mead_config)
        return config

    def read_config(self, config_params, datasets_index, **kwargs):
        """
        Read the config file and the datasets index

        Between the config file and the dataset index, we have enough information<commit_msg>fix: make sure feature names are unique (#356)<commit_after>import os
import json
import logging
from typing import List
from collections import Counter
import numpy as np
import eight_mile
import baseline
from copy import deepcopy
from baseline.reporting import create_reporting
from eight_mile.utils import (
    export,
    revlut,
    Offsets,
    get_env_gpus,
    import_user_module,
    listify,
)
from baseline.utils import (
    show_examples,
    normalize_backend,
)
from mead.downloader import EmbeddingDownloader, DataDownloader
from mead.utils import (
    index_by_label,
    get_mead_settings,
    print_dataset_info,
    read_config_file_or_json,
    get_dataset_from_key
)

from baseline.train import calc_lr_params

__all__ = []
exporter = export(__all__)
logger = logging.getLogger('mead')


def merge_reporting_with_settings(reporting, settings):
    default_reporting = settings.get('reporting_hooks', {})
    # Add default reporting information to the reporting settings.
    for report_type in default_reporting:
        if report_type in reporting:
            for report_arg, report_val in default_reporting[report_type].items():
                if report_arg not in reporting[report_type]:
                    reporting[report_type][report_arg] = report_val
    reporting_hooks = list(reporting.keys())
    for settings in reporting.values():
        for module in listify(settings.get('module', settings.get('modules', []))):
            import_user_module(module)
    return reporting_hooks, reporting



class Backend(object):
    """Simple object to represent a deep-learning framework backend"""
    def __init__(self, name=None, params=None, exporter=None):
        """Initialize the backend, optional with constructor args

        :param name: (``str``) Name of the framework: currently one of (`tensorflow`, `pytorch`)
        :param params: (``dict``) A dictionary of framework-specific user-data to pass through keyword args to each sub-module
        :param exporter: A framework-specific exporter to facilitate exporting to runtime deployment
        """
        self.name = normalize_backend(name)
        self.params = params

    def load(self, task_name=None):
        base_pkg_name = 'baseline.{}'.format(self.name)
        mod = import_user_module(base_pkg_name)
        import_user_module('eight_mile.{}.embeddings'.format(self.name))
        import_user_module('mead.{}.exporters'.format(self.name))
        if task_name is not None:
            import_user_module('{}.{}'.format(base_pkg_name, task_name))
        self.transition_mask = mod.transition_mask


TASK_REGISTRY = {}
@exporter
def register_task(cls):
    TASK_REGISTRY[cls.task_name()] = cls
    return cls

@exporter
def get_task_registry():
    return TASK_REGISTRY


def assert_unique_feature_names(names: List[str]) -> None:
    """Check if all the feature names are unique.

    :param names: The feature names
    :raises ValueError: If there are duplicated names
    """
    counts = Counter(names)
    dups = [n for n, c in counts.items() if c > 1]
    if dups:
        raise ValueError(f"Features names must be unique, found duplicates {dups}")


@exporter
class Task(object):
    """Basic building block for a task of NLP problems, e.g. `tagger`, `classify`, etc.
    """

    def _create_backend(self, **kwargs):
        """This method creates and returns a `Backend` object

        :return:
        """
        pass

    def __init__(self, mead_settings_config=None):
        super(Task, self).__init__()
        self.config_params = None
        self.mead_settings_config = get_mead_settings(mead_settings_config)
        if 'datacache' not in self.mead_settings_config:
            self.data_download_cache = os.path.expanduser("~/.bl-data")
            self.mead_settings_config['datacache'] = self.data_download_cache
        else:
            self.data_download_cache = os.path.expanduser(self.mead_settings_config['datacache'])
        logger.info("using %s as data/embeddings cache", self.data_download_cache)

    @classmethod
    def task_name(cls):
        """This classmethod returns the official name of this task, e.g., `classify` for classification

        :return: (``str``) - String name of this task
        """
        pass

    def _create_vectorizers(self):
        """Read the `features` section of the mead config.  This sections contains both embedding info and vectorizers
        Then use the vectorizer sub-section to instantiate the vectorizers and return them in a ``dict`` with name
        keyed off of the `features->name` and value of `vectorizer`

        :return: (``dict``) - A dictionary of the vectorizers keyed by feature name
        """
        self.vectorizers = {}

        features = self.config_params['features']
        assert_unique_feature_names([f['name'] for f in features])
        self.primary_key = features[0]['name']
        for feature in self.config_params['features']:
            key = feature['name']
            if '-' in key:
                raise ValueError('Feature names cannot contain "-". Found feature named "{}"'.format(key))
            if feature.get('primary', False) is True:
                self.primary_key = key
            vectorizer_section = feature.get('vectorizer', {'type': 'token1d'})
            vectorizer_section['mxlen'] = vectorizer_section.get('mxlen', self.config_params.get('preproc', {}).get('mxlen', -1))
            vectorizer_section['mxwlen'] = vectorizer_section.get('mxwlen', self.config_params.get('preproc', {}).get('mxwlen', -1))
            if 'transform' in vectorizer_section:
                vectorizer_section['transform_fn'] = eval(vectorizer_section['transform'])
            vectorizer = baseline.create_vectorizer(**vectorizer_section)
            self.vectorizers[key] = vectorizer


    @staticmethod
    def get_task_specific(task, mead_config):
        """Get the task from the task registry associated with the name

        :param task: The task name
        :return:
        """
        config = TASK_REGISTRY[task](mead_config)
        return config

    def read_config(self, config_params, datasets_index, **kwargs):
        """
        Read the config file and the datasets index

        Between the config file and the dataset index, we have enough information
{'fraction_non_alphanumeric': 0.09941181343716345, 'fraction_numerical': 0.0009941181343716345, 'mean_word_length': 3.1598897312198484, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7208179', 'n_tokens_mistral': 3434, 'n_tokens_neox': 3356, 'n_words': 950}