<commit_before># -*- coding: utf-8 -*-
'''
Entity transition probability
'''
from __future__ import print_function, division
from math import log

from entity_transition import EntityTransition
from entity_grid import EntityGrid
import utils


class EntityProbability(object):
    def __init__(self, text, coref=True):
        self._eg = EntityGrid(text).resolve_coreference(
        ) if coref else EntityGrid(text)
        self._coher_prob = self._coherence_prob()

    @property
    def grid(self):
        return self._eg.grid

    @property
    def coherence_prob(self):
        return self._coher_prob

    def _get_column_prob(self, col):
        column = self._eg.grid[col].tolist()
        sent_len = len(column)
        transition_count = {}
        for tran in zip(column[1:], column[:-1]):
            transition_count[tran] = transition_count.get(tran, 0) + 1
        probs = []
        for i, role in enumerate(column):
            if i == 0:
                probs.append(log(column.count(column[0]) / sent_len))
            else:
                tran_cnt = transition_count[(column[i], column[i - 1])]
                ent_cnt = column.count(column[i - 1])
                probs.append(log(tran_cnt / ent_cnt))
        assert all([p <= 0.0 for p in probs])
        return sum(probs) / len(probs)

    def _coherence_prob(self):
        res = []
        for col in self._eg.grid.columns:
            res.append(self._get_column_prob(col))
        return sum(res) / len(res)


if __name__ == '__main__':
    from pprint import pprint
    T1 = '''
        The Justice Department is conducting an anti-trust trial against Microsoft Corp with evidence that the company is increasingly attempting to crush competitors.
        Microsoft is accused of trying to forcefully buy into markets where its own products are not competitive enough to unseat established brands.
        The case revolves around evidence of Microsoft aggressively pressuring Netscape into merging browser software.
        Microsoft claims its tactics are commonplace and good economically.
        The government may file a civil suit ruling that conspiracy to curb competition through collusion is a violation of the Sherman Act.
        Microsoft continues to show increased earnings despite the trial.
        '''

    T2 = 'I have a friend called Bob. He loves playing basketball. I also love playing basketball. We play basketball together sometimes.'
    T3 = 'I like apple juice. He also likes it. He also likes playing basketball.'
    T4 = 'The Justice Department is conducting an anti-trust trial against\
              Microsoft Corp with evidence that the company is increasingly attempting to crush competitors.'

    T5 = 'You should heed my advice, people should be useing computers.Computers are an exelent way to comunicate.'

    T = T2
    e = EntityProbability(T)
    #pprint(e._eg.grid)
    pprint(e.coherence_prob)

    from pprint import pprint
    print(EntityProbability(T)._coherence_prob())
    print([('', EntityProbability(t)._coherence_prob())
           for t in utils.add_sents(T, 5, T5)])
    print([('', EntityProbability(t)._coherence_prob())
           for t in utils.remove_sents(T, 5)])
    print([('', EntityProbability(t)._coherence_prob())
           for t in utils.shuffle_sents(T, 5)])<commit_msg>Calculate transition prob of a corpus<commit_after># -*- coding: utf-8 -*-
'''
Entity transition probability
'''
from __future__ import print_function, division
from math import log

import numpy as np

from entity_transition import EntityTransition
from entity_grid import EntityGrid
import utils


class EntityProbability(object):
    def __init__(self, text, coref=True):
        self._eg = EntityGrid(text).resolve_coreference(
        ) if coref else EntityGrid(text)
        self._coher_prob = self._coherence_prob()

    @property
    def grid(self):
        return self._eg.grid

    @property
    def coherence_prob(self):
        return self._coher_prob

    def _get_column_prob(self, col):
        column = self._eg.grid[col].tolist()
        sent_len = len(column)
        assert sent_len > 1
        transition_count = {}
        for tran in zip(column[1:], column[:-1]):
            transition_count[tran] = transition_count.get(tran, 0) + 1
        probs = []
        for i, role in enumerate(column):
            if i == 0:
                probs.append(log(column.count(column[0]) / sent_len))
            else:
                tran_cnt = transition_count[(column[i], column[i - 1])]
                ent_cnt = column.count(column[i - 1])
                probs.append(log(tran_cnt / ent_cnt))
        assert all([p <= 0.0 for p in probs])
        return sum(probs) / len(probs)

    def _coherence_prob(self):
        res = []
        for col in self._eg.grid.columns:
            res.append(self._get_column_prob(col))
        return sum(res) / len(res)


class ProbabilityVector(object):
    def __init__(self, corpus):
        self._corpus = corpus
        self._probs = self._make_probs()

    @property
    def corpus(self):
        return self._corpus

    @property
    def probs(self):
        return self._probs

    @property
    def mean(self):
        return np.mean(self.probs)

    @property
    def std(self):
        return np.std(self.probs)

    @property
    def var(self):
        return np.var(self.probs)

    def _make_probs(self):
        res = []
        for text in self.corpus:
            p = EntityProbability(text).coherence_prob
            res.append(p)
        return res


if __name__ == '__main__':
    from pprint import pprint
    T1 = '''
        The Justice Department is conducting an anti-trust trial against Microsoft Corp with evidence that the company is increasingly attempting to crush competitors.
        Microsoft is accused of trying to forcefully buy into markets where its own products are not competitive enough to unseat established brands.
        The case revolves around evidence of Microsoft aggressively pressuring Netscape into merging browser software.
        Microsoft claims its tactics are commonplace and good economically.
        The government may file a civil suit ruling that conspiracy to curb competition through collusion is a violation of the Sherman Act.
        Microsoft continues to show increased earnings despite the trial.
        '''

    T2 = 'I have a friend called Bob. He loves playing basketball. I also love playing basketball. We play basketball together sometimes.'
    T3 = 'I like apple juice. He also likes it. He also likes playing basketball.'
    T4 = 'The Justice Department is conducting an anti-trust trial against\
              Microsoft Corp with evidence that the company is increasingly attempting to crush competitors.'

    T5 = 'You should heed my advice, people should be useing computers. Computers are an exelent way to comunicate.'

    T = T2
    e = EntityProbability(T)
    #pprint(e._eg.grid)
    pprint(e.coherence_prob)

    from pprint import pprint
    print(EntityProbability(T)._coherence_prob())
    print([('', EntityProbability(t)._coherence_prob())
           for t in utils.add_sents(T, 5, T5)])
    print([('', EntityProbability(t)._coherence_prob())
           for t in utils.remove_sents(T, 5)])
    print([('', EntityProbability(t)._coherence_prob())
           for t in utils.shuffle_sents(T, 5)])
    pv = ProbabilityVector([T1, T2, T3, T5])
    print(pv.probs, pv.mean, pv.var)
{'fraction_non_alphanumeric': 0.08766277352664788, 'fraction_numerical': 0.006309571754597933, 'mean_word_length': 2.730595893840761, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7227821', 'n_tokens_mistral': 2166, 'n_tokens_neox': 2045, 'n_words': 710}