<commit_before>#!/usr/bin/env python
# -*- mode: python; coding: utf-8 -*-

##################################################################
# Documentation
"""Module providing abstract base for all sentiment analyzers.

Attributes:
  BaseAnalyzer (class): abstract base for all sentiment analyzers

"""

##################################################################
# Imports
from __future__ import absolute_import, print_function, unicode_literals

from collections import defaultdict
from sklearn.model_selection import StratifiedKFold
try:
    from cPickle import load
except ImportError:
    from _pickle import load
import abc
import numpy as np
import os
import pandas as pd
import re

from cgsa.common import LOGGER
from cgsa.constants import ENCODING, NFOLDS

##################################################################
# Variables and Constants
TERM = "term"
POLCLASS = "polclass"
SCORE = "score"
LEX_CLMS = (TERM, POLCLASS, SCORE)
LEX_TYPES = {TERM: str, POLCLASS: str, SCORE: float}
NEG_SFX = r"_NEG"
NEG_SFX_RE = re.compile(re.escape(re.escape(NEG_SFX)
                                  + r"(:?FIRST)?'"))
USCORE_RE = re.compile(r'_')
SPACE_RE = re.compile(r"\s\s+")
AT_RE = re.compile(r"\b@\w+", re.U)
URI_RE = re.compile(
    r"\b((?:[\w]{3,5}://?|(?:www|bit)[.]|(?:\w[-\w]+[.])+"
    r"(?:a(?:ero|sia|[c-gil-oq-uwxz])|b(?:iz|[abd-jmnorstvwyz])"
    r"|c(?:at|o(?:m|op)|[acdf-ik-orsuvxyz])|d[dejkmoz]|e"
    r"(?:du|[ceghtu])|f[ijkmor]|g(?:ov|[abd-ilmnp-uwy])"
    r"|h[kmnrtu]|i(?:n(?:fo|t)|[del-oq-t])|j(?:obs|[emop])"
    r"|k[eghimnprwyz]|l[abcikr-vy]|m(?:il|obi|useum"
    r"|[acdeghk-z])|n(?:ame|et|[acefgilopruz])|"
    r"o(?:m|rg)|p(?:ro|[ae-hk-nrstwy])|qa|r[eosuw]"
    r"|s[a-eg-or-vxyz]|t(?:(?:rav)?el|[cdfghj-pr])"
    r"|xxx)\b)(?:[^\s,.:;]|\.\w)*)")
CI_PRFX = "%%case-insensitive-"


##################################################################
# Classes
class BaseAnalyzer(object):
    """Abstract class for coarse-grained sentiment analyzers.

    Attributes:

    """
    __metaclass__ = abc.ABCMeta

    @staticmethod
    def load(a_path):
        """Load model from disc and restore its members.

        Args:
          a_path (str): path to the serialized model

        """
        analyzer = load(a_path)
        analyzer._logger = LOGGER
        return analyzer

    def __init__(self, *args, **kwargs):
        """Class constructor.

        """
        self.name = "BaseAnalyzer"
        self._logger = LOGGER
        self._term2mnl = defaultdict(dict)
        self._neg_term2mnl = defaultdict(dict)
        self._term2auto = defaultdict(dict)
        self._neg_term2auto = defaultdict(dict)
        self._read_lexicons(self._term2mnl, self._neg_term2mnl,
                            kwargs.get("manual_lexicons"))
        self._read_lexicons(self._term2auto, self._neg_term2auto,
                            kwargs.get("auto_lexicons"))

    @abc.abstractmethod
    def train(self, train_x, train_y, dev_x, dev_y, grid_search=True):
        """Method for training the model.<commit_msg>provided default values (empty lists) for automatic and manual lexicon arguments<commit_after>#!/usr/bin/env python
# -*- mode: python; coding: utf-8 -*-

##################################################################
# Documentation
"""Module providing abstract base for all sentiment analyzers.

Attributes:
  BaseAnalyzer (class): abstract base for all sentiment analyzers

"""

##################################################################
# Imports
from __future__ import absolute_import, print_function, unicode_literals

from collections import defaultdict
from sklearn.model_selection import StratifiedKFold
try:
    from cPickle import load
except ImportError:
    from _pickle import load
import abc
import numpy as np
import os
import pandas as pd
import re

from cgsa.common import LOGGER
from cgsa.constants import ENCODING, NFOLDS

##################################################################
# Variables and Constants
TERM = "term"
POLCLASS = "polclass"
SCORE = "score"
LEX_CLMS = (TERM, POLCLASS, SCORE)
LEX_TYPES = {TERM: str, POLCLASS: str, SCORE: float}
NEG_SFX = r"_NEG"
NEG_SFX_RE = re.compile(re.escape(re.escape(NEG_SFX)
                                  + r"(:?FIRST)?'"))
USCORE_RE = re.compile(r'_')
SPACE_RE = re.compile(r"\s\s+")
AT_RE = re.compile(r"\b@\w+", re.U)
URI_RE = re.compile(
    r"\b((?:[\w]{3,5}://?|(?:www|bit)[.]|(?:\w[-\w]+[.])+"
    r"(?:a(?:ero|sia|[c-gil-oq-uwxz])|b(?:iz|[abd-jmnorstvwyz])"
    r"|c(?:at|o(?:m|op)|[acdf-ik-orsuvxyz])|d[dejkmoz]|e"
    r"(?:du|[ceghtu])|f[ijkmor]|g(?:ov|[abd-ilmnp-uwy])"
    r"|h[kmnrtu]|i(?:n(?:fo|t)|[del-oq-t])|j(?:obs|[emop])"
    r"|k[eghimnprwyz]|l[abcikr-vy]|m(?:il|obi|useum"
    r"|[acdeghk-z])|n(?:ame|et|[acefgilopruz])|"
    r"o(?:m|rg)|p(?:ro|[ae-hk-nrstwy])|qa|r[eosuw]"
    r"|s[a-eg-or-vxyz]|t(?:(?:rav)?el|[cdfghj-pr])"
    r"|xxx)\b)(?:[^\s,.:;]|\.\w)*)")
CI_PRFX = "%%case-insensitive-"


##################################################################
# Classes
class BaseAnalyzer(object):
    """Abstract class for coarse-grained sentiment analyzers.

    Attributes:

    """
    __metaclass__ = abc.ABCMeta

    @staticmethod
    def load(a_path):
        """Load model from disc and restore its members.

        Args:
          a_path (str): path to the serialized model

        """
        analyzer = load(a_path)
        analyzer._logger = LOGGER
        return analyzer

    def __init__(self, *args, **kwargs):
        """Class constructor.

        """
        self.name = "BaseAnalyzer"
        self._logger = LOGGER
        self._term2mnl = defaultdict(dict)
        self._neg_term2mnl = defaultdict(dict)
        self._term2auto = defaultdict(dict)
        self._neg_term2auto = defaultdict(dict)
        self._read_lexicons(self._term2mnl, self._neg_term2mnl,
                            kwargs.get("manual_lexicons", []))
        self._read_lexicons(self._term2auto, self._neg_term2auto,
                            kwargs.get("auto_lexicons", []))

    @abc.abstractmethod
    def train(self, train_x, train_y, dev_x, dev_y, grid_search=True):
        """Method for training the model.
{'fraction_non_alphanumeric': 0.23950737319721277, 'fraction_numerical': 0.0035650623885918, 'mean_word_length': 3.707856598016781, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6382701', 'n_tokens_mistral': 2229, 'n_tokens_neox': 2156, 'n_words': 395}