<commit_before>

import os
import re
from math import log1p
import operator

from cheshire3.document import StringDocument
from cheshire3.internal import cheshire3Root
from cheshire3.server import SimpleServer
from cheshire3.baseObjects import Session

cheshirePath = os.path.join('HOME', '/home/cheshire')

class Keywords(object):
    
    def __init__(self):
        self.session = Session()
        self.session.database = 'db_dickens'
        self.serv = SimpleServer(self.session,
                            os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')
                            )
        self.db = self.serv.get_object(self.session, self.session.database)
        self.qf = self.db.get_object(self.session, 'defaultQueryFactory')
        self.resultSetStore = self.db.get_object(self.session, 'resultSetStore')        
        self.idxStore = self.db.get_object(self.session, 'indexStore')
        self.logger = self.db.get_object(self.session, 'keywordLogger')
        
    def list_keywords(self, testIdxName, testMaterials, refIdxName, refMaterials, pValue):
        #self.logger.log(10, 'CREATING KEYWORDS FOR RS: {0} in {1}, compared to {2} in {3}'.format(testIdxName, testMaterials, refIdxName, refMaterials)) 
        session = self.session
        db = self.db

        # Test results
        clauses = []
        for testMaterial in testMaterials:
            if testMaterial in ['dickens', 'ntc']:
                testMatIdx = 'subCorpus-idx'
            else:
                testMatIdx = 'book-idx'
            clauses.append('c3.{0} = "{1}"'.format(testMatIdx, testMaterial))
            
        test_query = self.qf.get_query(session,
                                       ' or '.join(clauses)
                                       )        
        test_results = db.search(session, test_query)
        test_idx = db.get_object(session, testIdxName)
        test_facets = test_idx.facets(session, test_results)
        ## create dictionary containing word/cluster and number of occurrences
        test_dict = {}
        for x in test_facets:
            test_dict[x[0]] = x[1][2]        
        
        # Reference results
        clauses_ref = []
        for refMaterial in refMaterials:
            if refMaterial in ['dickens', 'ntc']:
                refMatIdx = 'subCorpus-idx'
            else:
                refMatIdx = 'book-idx'
            clauses_ref.append('c3.{0} = "{1}"'.format(refMatIdx, refMaterial))

        ref_query = self.qf.get_query(session,
                                       ' or '.join(clauses_ref)
                                       )
        ref_results = db.search(session, ref_query)
        ref_idx = db.get_object(session, refIdxName)
        ref_facets = ref_idx.facets(session, ref_results)
        #ref_dict = {x[0]: x[1][2] for x in ref_facets}
        ref_dict = {}
        for x in ref_facets:
            ref_dict[x[0]] = x[1][2]
        
        ## get test and ref lengths
        ## I use total counts to calculate expected values
        testLength = sum(test_dict.values())
        refLength = sum(ref_dict.values())

        
        kw_list = []
        for term, freqTest in test_dict.iteritems():
            if freqTest > 1:
                try:
                    ## Method 1: how many observations of a given word is found in ref corpus but not in test corpus
                    ## Subtract number of occurrences in testIndex from number of occurrences in sentences
                    #freqRef = float(ref_dict[term] - freqTest)
                    ## Method 2: treat groups as mutually exclusive. NOTE: When comparing quotes with whole text the occurrences will overlap
                    freqRef = float(ref_dict[term])
                except KeyError:
                    freqRef = 5.0e-324
                else:
                    if freqRef <= 0:
                        freqRef = 5.0e-324
                
                ## following Paul Ryson formula for log likelihood (http://ucrel.lancs.ac.uk/llwizard.html)
                ## 1. Expected occurrence within corpus
                ## 1a. Expected reference value: based on sentence index
                ## - Get the total N from corpus 1 (reference corpus)
                ## - Multiply by the sum of observations found in ref corpus and those found in test corpus
                ## - Divide by the sum of total N in test corpus and reference corpus
                expectedRef = refLength*(freqTest+freqRef)/(testLength+refLength)
                ## 1b. Expected test value
                ## Equivalent steps to 1a, but multiply by test N
                expectedTest = testLength*(freqTest+freqRef)/(testLength+refLength)
                  
                ## 2. Log Likelihood
                ## Compare actual observations with expected ocurrence for both test and ref, and add these values
                ## Use log1p() (for natural logarithm - ln) instead of log()<commit_msg>more efficient looping across ngrams<commit_after>

import os
import re
from math import log1p
import operator

from cheshire3.document import StringDocument
from cheshire3.internal import cheshire3Root
from cheshire3.server import SimpleServer
from cheshire3.baseObjects import Session

cheshirePath = os.path.join('HOME', '/home/cheshire')

class Keywords(object):
    
    def __init__(self):
        self.session = Session()
        self.session.database = 'db_dickens'
        self.serv = SimpleServer(self.session,
                            os.path.join(cheshire3Root, 'configs', 'serverConfig.xml')
                            )
        self.db = self.serv.get_object(self.session, self.session.database)
        self.qf = self.db.get_object(self.session, 'defaultQueryFactory')
        self.resultSetStore = self.db.get_object(self.session, 'resultSetStore')        
        self.idxStore = self.db.get_object(self.session, 'indexStore')
        self.logger = self.db.get_object(self.session, 'keywordLogger')
        
    def list_keywords(self, testIdxName, testMaterials, refIdxName, refMaterials, pValue):
        #self.logger.log(10, 'CREATING KEYWORDS FOR RS: {0} in {1}, compared to {2} in {3}'.format(testIdxName, testMaterials, refIdxName, refMaterials)) 
        session = self.session
        db = self.db

        # Test results
        clauses = []
        for testMaterial in testMaterials:
            if testMaterial in ['dickens', 'ntc']:
                testMatIdx = 'subCorpus-idx'
            else:
                testMatIdx = 'book-idx'
            clauses.append('c3.{0} = "{1}"'.format(testMatIdx, testMaterial))
            
        test_query = self.qf.get_query(session,
                                       ' or '.join(clauses)
                                       )        
        test_results = db.search(session, test_query)
        test_idx = db.get_object(session, testIdxName)
        test_facets = test_idx.facets(session, test_results)
        ## create dictionary containing word/cluster and number of occurrences
        test_dict = {x[0]: x[1][2] for x in test_facets}
#         test_dict = {}
#         for x in test_facets:
#             test_dict[x[0]] = x[1][2]        
        
        # Reference results
        clauses_ref = []
        for refMaterial in refMaterials:
            if refMaterial in ['dickens', 'ntc']:
                refMatIdx = 'subCorpus-idx'
            else:
                refMatIdx = 'book-idx'
            clauses_ref.append('c3.{0} = "{1}"'.format(refMatIdx, refMaterial))

        ref_query = self.qf.get_query(session,
                                       ' or '.join(clauses_ref)
                                       )
        ref_results = db.search(session, ref_query)
        ref_idx = db.get_object(session, refIdxName)
        ref_facets = ref_idx.facets(session, ref_results)
        ref_dict = {x[0]: x[1][2] for x in ref_facets}
#         ref_dict = {}
#         for x in ref_facets:
#             ref_dict[x[0]] = x[1][2]
        
        ## get test and ref lengths
        ## I use total counts to calculate expected values
        testLength = sum(test_dict.values())
        refLength = sum(ref_dict.values())        
        
        kw_list = []
        for term, freqTest in test_dict.iteritems():
            if freqTest > 1:
                try:
                    ## Method 1: how many observations of a given word is found in ref corpus but not in test corpus
                    ## Subtract number of occurrences in testIndex from number of occurrences in sentences
                    #freqRef = float(ref_dict[term] - freqTest)
                    ## Method 2: treat groups as mutually exclusive. NOTE: When comparing quotes with whole text the occurrences will overlap
                    freqRef = float(ref_dict[term])
                except KeyError:
                    freqRef = 5.0e-324
                else:
                    if freqRef <= 0:
                        freqRef = 5.0e-324
                
                ## following <NAME> formula for log likelihood (http://ucrel.lancs.ac.uk/llwizard.html)
                ## 1. Expected occurrence within corpus
                ## 1a. Expected reference value: based on sentence index
                ## - Get the total N from corpus 1 (reference corpus)
                ## - Multiply by the sum of observations found in ref corpus and those found in test corpus
                ## - Divide by the sum of total N in test corpus and reference corpus
                expectedRef = refLength*(freqTest+freqRef)/(testLength+refLength)
                ## 1b. Expected test value
                ## Equivalent steps to 1a, but multiply by test N
                expectedTest = testLength*(freqTest+freqRef)/(testLength+refLength)
                  
                ## 2. Log Likelihood
                ## Compare actual observations with expected ocurrence for both test and ref, and add these values
                ## Use log1p() (for natural logarithm - ln) instead of log()
{'fraction_non_alphanumeric': 0.08881118881118881, 'fraction_numerical': 0.01008991008991009, 'mean_word_length': 1.7987140061504054, 'pattern_counts': {'":': 0, '<': 6, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 14, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5490284', 'n_tokens_mistral': 2841, 'n_tokens_neox': 2661, 'n_words': 757}