<commit_before>"""
lexical chain module for text tiling
"""

from tile_reader import TileReader
from scoring import boundarize, depth_scoring, window_diff


# ======================================================================================================================
# Main
# ======================================================================================================================
class LexicalChains(object):

    def __init__(self):
        self.sentences = []
        self.actives = {}
        self.gap_scores = []
        self.boundary_vector = []

    def analyze(self, sents, window=4, pos_filter=('PUNCT', 'SYM', 'SPACE'), boundary_type='liberal'):
        """
        Set attributes
        :param sents: (list) spacy-analyzed sentences
        :param window: (int) distance threshold within which chains are considered active
        :param boundary_type: (str) 'liberal' or 'conservative' boundary scoring
        :param pos_filter: (tuple) spacy pos_ labels to exclude (i.e. a pos-based stoplist)
        :return: void
        """
        self.sentences = self.__preproc(sents, pos_filter)
        self.actives = self.__get_actives(self.sentences, window)
        self.gap_scores = [len(self.actives[k]) for k in self.actives.keys()]
        self.boundary_vector = self.__get_boundaries(self.gap_scores, boundary_type)

    @staticmethod
    def __preproc(sentences, pos_filter):
        """
        Filters out stop POSs and lemmatizes sentences
        :param sentences: list of tokenized sentences in doc
        :param pos_filter: tuple of spacy pos_ labels to filter out
        :return: list
        """
        filtered = [[tok for tok in sent if tok.pos_ not in pos_filter] for sent in sentences]
        lemmatized = [[tok.lemma_ for tok in sent] for sent in filtered]
        return lemmatized

    @staticmethod
    def __get_actives(sents, window):
        """
        Get active lexical chains for each gap between sentences
        :param sents: list of tokenized sentences
        :param window: difference threshold over which lexical chains are considered active
        :return: dictionary containing active lexical chains for each sentence transition
        """
        # initialize active chains dictionary
        actives = {}
        for i in xrange(len(sents)-1):
            actives[i] = set()

        # loop over all sentences
        for sent in sents:
            # get index and unique tokens from current sentence
            i = sents.index(sent)
            uniques_i = set(sent)

            # loop over all sentences within dist thresh of current
            for diff in xrange(window, 0, -1):
                # back off diff when there are less sentences left than dist thresh
                while not i + diff < len(sents):
                    diff -= 1
                # find shared tokens between current sent[i] and sent[i+diff]
                n = i + diff
                uniques_n = set(sents[n])
                intersection = uniques_i.intersection(uniques_n)

                # add the intersections to all affected transitions between sent[i] and sent[i+diff]
                for k in list(xrange(diff)):
                    [actives[i+k].add(word) for word in intersection]

        return actives

    @staticmethod
    def __get_boundaries(scores, boundary_type):
        """
        Calculate boundaries from gap scores
        :param scores: list containing # of active chains across each sentence gap in doc
        :param boundary_type: string indicating 'liberal' or 'conservative' boundary scoring
        :return: list indicating which sentences in doc constitute beginnings of new topic tiles
        """
        d_scores = depth_scoring(scores)
        boundaries = boundarize(d_scores, type=boundary_type)
        boundary_vector = [1] + [0 if i not in boundaries else 1 for i in xrange(len(scores))]
        return boundary_vector


# ======================================================================================================================
# Run
# ======================================================================================================================
if __name__ == "__main__":
    from decimal import Decimal
    import matplotlib.pyplot as plt
    import sys

    # set doc
    try:
        doc = sys.argv[1]
    except IndexError:
        sys.exit("ERROR: Expected 1 arg, got {}\nUsage: (python) lexical_chains.py <docname>".format(len(sys.argv)-1))

    # get path
    if doc in ('coron','athens','chatham','cuba','merida'):
        doc_path = r"C:\Users\Sean\Documents\_University\2016_fall\comp_disc\compdisc\data\GUM_voyage_{}_noheads.txt".format(doc)
    elif doc == "robot":
        doc_path = r"C:\Users\Sean\Documents\_University\2016_fall\comp_disc\compdisc\data\robot.txt"<commit_msg>allow docpath to be specified by user when invoked directly<commit_after>"""
lexical chain module for text tiling
"""

from tile_reader import TileReader
from scoring import boundarize, depth_scoring, window_diff


# ======================================================================================================================
# Main
# ======================================================================================================================
class LexicalChains(object):

    def __init__(self):
        self.sentences = []
        self.actives = {}
        self.gap_scores = []
        self.boundary_vector = []

    def analyze(self, sents, window=4, pos_filter=('PUNCT', 'SYM', 'SPACE'), boundary_type='liberal'):
        """
        Set attributes
        :param sents: (list) spacy-analyzed sentences
        :param window: (int) distance threshold within which chains are considered active
        :param boundary_type: (str) 'liberal' or 'conservative' boundary scoring
        :param pos_filter: (tuple) spacy pos_ labels to exclude (i.e. a pos-based stoplist)
        :return: void
        """
        self.sentences = self.__preproc(sents, pos_filter)
        self.actives = self.__get_actives(self.sentences, window)
        self.gap_scores = [len(self.actives[k]) for k in self.actives.keys()]
        self.boundary_vector = self.__get_boundaries(self.gap_scores, boundary_type)

    @staticmethod
    def __preproc(sentences, pos_filter):
        """
        Filters out stop POSs and lemmatizes sentences
        :param sentences: list of tokenized sentences in doc
        :param pos_filter: tuple of spacy pos_ labels to filter out
        :return: list
        """
        filtered = [[tok for tok in sent if tok.pos_ not in pos_filter] for sent in sentences]
        lemmatized = [[tok.lemma_ for tok in sent] for sent in filtered]
        return lemmatized

    @staticmethod
    def __get_actives(sents, window):
        """
        Get active lexical chains for each gap between sentences
        :param sents: list of tokenized sentences
        :param window: difference threshold over which lexical chains are considered active
        :return: dictionary containing active lexical chains for each sentence transition
        """
        # initialize active chains dictionary
        actives = {}
        for i in xrange(len(sents)-1):
            actives[i] = set()

        # loop over all sentences
        for sent in sents:
            # get index and unique tokens from current sentence
            i = sents.index(sent)
            uniques_i = set(sent)

            # loop over all sentences within dist thresh of current
            for diff in xrange(window, 0, -1):
                # back off diff when there are less sentences left than dist thresh
                while not i + diff < len(sents):
                    diff -= 1
                # find shared tokens between current sent[i] and sent[i+diff]
                n = i + diff
                uniques_n = set(sents[n])
                intersection = uniques_i.intersection(uniques_n)

                # add the intersections to all affected transitions between sent[i] and sent[i+diff]
                for k in list(xrange(diff)):
                    [actives[i+k].add(word) for word in intersection]

        return actives

    @staticmethod
    def __get_boundaries(scores, boundary_type):
        """
        Calculate boundaries from gap scores
        :param scores: list containing # of active chains across each sentence gap in doc
        :param boundary_type: string indicating 'liberal' or 'conservative' boundary scoring
        :return: list indicating which sentences in doc constitute beginnings of new topic tiles
        """
        d_scores = depth_scoring(scores)
        boundaries = boundarize(d_scores, type=boundary_type)
        boundary_vector = [1] + [0 if i not in boundaries else 1 for i in xrange(len(scores))]
        return boundary_vector


# ======================================================================================================================
# Test if invoked directly
# ======================================================================================================================
if __name__ == "__main__":
    from decimal import Decimal
    import matplotlib.pyplot as plt
    import sys
    import os

    # set doc
    try:
        doc = sys.argv[1]
        docpath = sys.argv[2]
    except IndexError:
        sys.exit("ERROR: Expected 2 arg, got {}\nUsage: (python) lexical_chains.py <docname> <docpath>".format(
            len(sys.argv)-1))

    # get path
    if doc in ('coron','athens','chatham','cuba','merida'):
        doc_path = os.path.join(docpath, "GUM_voyage_{}_noheads.txt".format(doc))
    elif doc == "robot":
        doc_path = os.path.join(docpath, "robot.txt")
{'fraction_non_alphanumeric': 0.17588769611891, 'fraction_numerical': 0.003199834847233691, 'mean_word_length': 2.8281311734492296, 'pattern_counts': {'":': 4, '<': 8, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1021591', 'n_tokens_mistral': 2599, 'n_tokens_neox': 2426, 'n_words': 813}