<commit_before># -*- coding: utf-8 -*-
#!/usr/bin/python

from collections import Counter
import re
import os
import pickle

import numpy as np
from sklearn.decomposition import PCA

from nltk.tokenize import word_tokenize, sent_tokenize
from nltk import Tree

import utils
import utils_wordlists
import utils_parsing
import utils_similarity
from utils_vsm import semantic_orientation_score_lexicon


GLOVE_SIZE = 50 # 50 is appropriate -- bigger starts to seriously 
                # outpace words per paragraph
path_to_glove = os.environ.get('GLV_HOME')
if not path_to_glove:
  raise ImportError("GLV_HOME not defined as environmental variable")
glove = None

semantic_lexicon = None
if os.path.exists(os.path.join('data/', 'semantic_lexicon.pickle')):
  semantic_lexicon = pickle.load(file(os.path.join('data/', 'semantic_lexicon.pickle')))

# Note: Best to use independent namespaces for each key,
# since multiple feature functions can be grouped together.



###########################################
# Master feature functions:
###########################################

def baseline_features(paragraph, parse):
  return manual_content_flags(paragraph, parse) | \
    baseline_length(paragraph, parse)

def simple_features(paragraph, parse):
  return lexical_features(paragraph, parse) | \
    length(paragraph, parse) | \
    syntactic_features(paragraph, parse)

def semcom_sentiment_features(paragraph, parse):
  return word_intensity(paragraph, parse)

def semcom_pca_features(paragraph, unused_parse):
  # As per experimental results, cumulative + 10 eigenvalues is best
  return dimensional_decomposition(paragraph, unused_parse, 10)

def semcom_lstm_features(paragraph, unused_parse):
  raise ValueError("LSTM features not yet implemented")

def semcom_ka_features(paragraph, parse):
  return kannan_ambili(paragraph, parse)

def all_features(paragraph, parse):
  return baseline_features(paragraph, parse) | \
    lexical_features(paragraph, parse) | \
    length(paragraph, parse) | \
    syntactic_features(paragraph, parse) | \
    semcom_sentiment_features(paragraph, parse) | \
    semcom_pca_features(paragraph, parse) | \
    semcom_lstm_features(paragraph, parse) | \
    semcom_ka_features(paragraph, parse)

##########################################
# Sub-units (for ablation)
# - lexicographic features (= length)
# - lexical features (particular word categories)
# - syntactic features (determiners & parse complexity)
##########################################
def lexical_features(paragraph, parse):
  return get_morphological_counts(paragraph, parse) | \
    modal_presence(paragraph, parse) | \
    relative_amount_presence(paragraph, parse) | \
    transitional_presence(paragraph, parse) | \
    hedge_presence(paragraph, parse) | \
    conjunctives_presence(paragraph, parse) | \
    punctuation_presence(paragraph, parse)

def syntactic_features(paragraph, parse):
  return determiner_usage(paragraph, parse) | \
    syntactic_parse_features(paragraph, parse)


###########################################
# Components:
###########################################

def get_morphological_counts(paragraph, unused_parse):
  """
  Return a counter containing:
   number of times '-----er' occurs
   number of times '-----ly' occurs
   number of times '-----est' occurs

  Parameters
  ----------
  paragraph : string
      Content string from which features should be extracted.

  Returns
  -------
  dict : string -> integer
  """
  features = Counter()
  tokenized_and_lowercase = word_tokenize(paragraph.lower())
  er_count = 0
  ly_count = 0
  est_count = 0
  for w in tokenized_and_lowercase:  
    if w.endswith('er'):
      er_count += 1
    elif w.endswith("est"):
      est_count += 1
    elif w.endswith("ly"):
      ly_count += 1

  features['er_count'] = er_count
  features['est_count'] = est_count
  features['ly_count'] = ly_count

  return features


def word_intensity(paragraph, unused_parse):
    """
    Return level of word intensity in the paragraph based on semantic orientation.
    Code adapted from word similarity class

    Parameters
    ----------
    paragraph : string
        Content string from which features should be extracted.

    Returns
    -------
    dict : string -> float
    """
    AVG_SENTENCE_LENGTH = 15

    ##   Reading in locally cached Glove File
    global semantic_lexicon
    if semantic_lexicon == None:
      glv = utils.build_glove(os.path.join(path_to_glove, 
              'glove.6B.%dd.txt' % GLOVE_SIZE))
      semantic_lexicon = semantic_orientation_score_lexicon(mat=glv[0], rownames=glv[1],
        negset=('little', 'few', 'never', 'low', 'negative', 'wrong', 'small', 'inferior', 'seldom'),
        posset=('extremely', 'many', 'always', 'high', 'positive', 'correct', 'big', 'superior', 'often'))
      pickle.dump(semantic_lexicon, open( "data/semantic_lexicon.pickle", "w+" ) )

    ##   Calculate word and sentence sentiment variance
    intensity = []
    sentence = []
    sentence_means = []
    tokenized_and_lowercase = word_tokenize(paragraph.lower())
    punctuated = "." in tokenized_and_lowercase    #   Use this to account for punctuation
    #   punctuated = False    #   Use this when dataset has no punctuation
    for word in tokenized_and_lowercase:
        if word in semantic_lexicon:
            intensity.append(semantic_lexicon[word])
            sentence.append(semantic_lexicon[word])
        if (not punctuated and len(sentence) >= AVG_SENTENCE_LENGTH) \
        or (punctuated and word  in [".", "?", "!"]):
            m = np.mean(sentence)
            sentence_means.append(m)
            sentence = []
    m = np.mean(sentence)
    sentence_means.append(m)
    sentence_var = 0         # 0 variance when there's only 1 sentence
    if len(sentence_means) > 1:
        sentence_var = np.var(sentence_means)
    # print {'all_var': np.var(intensity), 'sentence_var': sentence_var}
    max_intensity = max(intensity)
    min_intensity = min(intensity)
    # return Counter({'all_var': np.var(intensity), 'sentence_var': sentence_var})
    return Counter({'min_max': max_intensity - min_intensity})

def manual_content_flags(paragraph, unused_parse):
    """
    Baseline feature extractor, based on manual. Produces a feature function 
    that detects the presence of the example "content flag" phrases in the 
    coding manual.
    
    Parameters
    ----------
    paragraph : string
        Content string from which features should be extracted.
              
    Returns
    -------
    dict : string -> integer
        The number of times a flag at each level occurred in the text 
        (e.g., {"flag_1": 3, "flag_2": 1} )
      
    """<commit_msg>Fixes bug in sentiment: in old cold, if sentence ends in a period of multiple of AVG_SENTENCE_LENGTH a bug arose related to mean of empty sentence vector<commit_after># -*- coding: utf-8 -*-
#!/usr/bin/python

from collections import Counter
import re
import os
import pickle

import numpy as np
from sklearn.decomposition import PCA

from nltk.tokenize import word_tokenize, sent_tokenize
from nltk import Tree

import utils
import utils_wordlists
import utils_parsing
import utils_similarity
from utils_vsm import semantic_orientation_score_lexicon


GLOVE_SIZE = 50 # 50 is appropriate -- bigger starts to seriously 
                # outpace words per paragraph
path_to_glove = os.environ.get('GLV_HOME')
if not path_to_glove:
  raise ImportError("GLV_HOME not defined as environmental variable")
glove = None

semantic_lexicon = None
if os.path.exists(os.path.join('data/', 'semantic_lexicon.pickle')):
  semantic_lexicon = pickle.load(file(os.path.join('data/', 'semantic_lexicon.pickle')))

# Note: Best to use independent namespaces for each key,
# since multiple feature functions can be grouped together.



###########################################
# Master feature functions:
###########################################

def baseline_features(paragraph, parse):
  return manual_content_flags(paragraph, parse) | \
    baseline_length(paragraph, parse)

def simple_features(paragraph, parse):
  return lexical_features(paragraph, parse) | \
    length(paragraph, parse) | \
    syntactic_features(paragraph, parse)

def semcom_sentiment_features(paragraph, parse):
  return word_intensity(paragraph, parse)

def semcom_pca_features(paragraph, unused_parse):
  # As per experimental results, cumulative + 10 eigenvalues is best
  return dimensional_decomposition(paragraph, unused_parse, 10)

def semcom_lstm_features(paragraph, unused_parse):
  return Counter() #raise ValueError("LSTM features not yet implemented")

def semcom_ka_features(paragraph, parse):
  return kannan_ambili(paragraph, parse)

def all_features(paragraph, parse):
  return baseline_features(paragraph, parse) | \
    lexical_features(paragraph, parse) | \
    length(paragraph, parse) | \
    syntactic_features(paragraph, parse) | \
    semcom_sentiment_features(paragraph, parse) | \
    semcom_pca_features(paragraph, parse) | \
    semcom_lstm_features(paragraph, parse) | \
    semcom_ka_features(paragraph, parse)

##########################################
# Sub-units (for ablation)
# - lexicographic features (= length)
# - lexical features (particular word categories)
# - syntactic features (determiners & parse complexity)
##########################################
def lexical_features(paragraph, parse):
  return get_morphological_counts(paragraph, parse) | \
    modal_presence(paragraph, parse) | \
    relative_amount_presence(paragraph, parse) | \
    transitional_presence(paragraph, parse) | \
    hedge_presence(paragraph, parse) | \
    conjunctives_presence(paragraph, parse) | \
    punctuation_presence(paragraph, parse)

def syntactic_features(paragraph, parse):
  return determiner_usage(paragraph, parse) | \
    syntactic_parse_features(paragraph, parse)


###########################################
# Components:
###########################################

def get_morphological_counts(paragraph, unused_parse):
  """
  Return a counter containing:
   number of times '-----er' occurs
   number of times '-----ly' occurs
   number of times '-----est' occurs

  Parameters
  ----------
  paragraph : string
      Content string from which features should be extracted.

  Returns
  -------
  dict : string -> integer
  """
  features = Counter()
  tokenized_and_lowercase = word_tokenize(paragraph.lower())
  er_count = 0
  ly_count = 0
  est_count = 0
  for w in tokenized_and_lowercase:  
    if w.endswith('er'):
      er_count += 1
    elif w.endswith("est"):
      est_count += 1
    elif w.endswith("ly"):
      ly_count += 1

  features['er_count'] = er_count
  features['est_count'] = est_count
  features['ly_count'] = ly_count

  return features


def word_intensity(paragraph, unused_parse):
    """
    Return level of word intensity in the paragraph based on semantic orientation.
    Code adapted from word similarity class

    Parameters
    ----------
    paragraph : string
        Content string from which features should be extracted.

    Returns
    -------
    dict : string -> float
    """
    AVG_SENTENCE_LENGTH = 15

    ##   Reading in locally cached Glove File
    global semantic_lexicon
    if semantic_lexicon == None:
      glv = utils.build_glove(os.path.join(path_to_glove, 
              'glove.6B.%dd.txt' % GLOVE_SIZE))
      semantic_lexicon = semantic_orientation_score_lexicon(mat=glv[0], rownames=glv[1],
        negset=('little', 'few', 'never', 'low', 'negative', 'wrong', 'small', 'inferior', 'seldom'),
        posset=('extremely', 'many', 'always', 'high', 'positive', 'correct', 'big', 'superior', 'often'))
      pickle.dump(semantic_lexicon, open( "data/semantic_lexicon.pickle", "w+" ) )

    ##   Calculate word and sentence sentiment variance
    intensity = []
    sentence = []
    sentence_means = []
    tokenized_and_lowercase = word_tokenize(paragraph.lower())
    punctuated = "." in tokenized_and_lowercase    #   Use this to account for punctuation
    #   punctuated = False    #   Use this when dataset has no punctuation
    for word in tokenized_and_lowercase:
        if word in semantic_lexicon:
            intensity.append(semantic_lexicon[word])
            sentence.append(semantic_lexicon[word])
        if (not punctuated and len(sentence) >= AVG_SENTENCE_LENGTH) \
        or (punctuated and word  in [".", "?", "!"]):
            m = np.mean(sentence)
            sentence_means.append(m)
            sentence = []
    if len(sentence) > 0: # Capture any leftovers
        print sentence
        m = np.mean(sentence)
        sentence_means.append(m)
    sentence_var = 0         # 0 variance when there's only 1 sentence
    if len(sentence_means) > 1:
        sentence_var = np.var(sentence_means)
    # print {'all_var': np.var(intensity), 'sentence_var': sentence_var}
    max_intensity = max(intensity)
    min_intensity = min(intensity)
    # return Counter({'all_var': np.var(intensity), 'sentence_var': sentence_var})
    return Counter({'min_max': max_intensity - min_intensity})

def manual_content_flags(paragraph, unused_parse):
    """
    Baseline feature extractor, based on manual. Produces a feature function 
    that detects the presence of the example "content flag" phrases in the 
    coding manual.
    
    Parameters
    ----------
    paragraph : string
        Content string from which features should be extracted.
              
    Returns
    -------
    dict : string -> integer
        The number of times a flag at each level occurred in the text 
        (e.g., {"flag_1": 3, "flag_2": 1} )
      
    """
{'fraction_non_alphanumeric': 0.13773377337733772, 'fraction_numerical': 0.004180418041804181, 'mean_word_length': 3.6843009275163174, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7531380', 'n_tokens_mistral': 4085, 'n_tokens_neox': 3870, 'n_words': 1064}