<commit_before>from dragonfire.utilities import Data
import wikipedia
import spacy
import collections
from nltk.corpus import wordnet as wn
import random

class Engine():

    def __init__(self):
        self.nlp = spacy.load('en')
        self.entity_map = {
                'WHO': ['PERSON'],
                'WHAT': ['PERSON','NORP','FACILITY','ORG','GPE','LOC','PRODUCT','EVENT','WORK_OF_ART','LANGUAGE','DATE','TIME','PERCENT','MONEY','QUANTITY','ORDINAL','CARDINAL'],
                'WHEN': ['DATE','TIME','EVENT'],
                'WHERE': ['FACILITY','GPE','LOC']
        }
        self.coefficient = {'frequency': 0.27, 'precedence': 0.27, 'proximity': 0.23, 'mention': 0.23}

    def respond(self,com,tts_output=False):
        userin = Data([" "]," ")
        doc = self.nlp(com.decode('utf-8'))
        query = None
        subjects = []
        pobjects = []
        dobjects = []
        for np in doc.noun_chunks:
            #print(np.text, np.root.text, np.root.dep_, np.root.head.text)
            if (np.root.dep_ == 'nsubj' or np.root.dep_ == 'nsubjpass') and np.root.tag_ != 'WP':
                subjects.append(np.text.encode('utf-8'))
            if np.root.dep_ == 'pobj':
                pobjects.append(np.text.encode('utf-8'))
            if np.root.dep_ == 'dobj':
                dobjects.append(np.text.encode('utf-8'))
        if pobjects:
            query = ' '.join(pobjects)
        elif subjects:
            query = ' '.join(subjects)
        elif dobjects:
            query = ' '.join(dobjects)
        else:
            if not tts_output: print "Sorry, I don't understand the subject of your question."
            if tts_output: userin.say("Sorry, I don't understand the subject of your question.")
            return False

        focus = None
        if dobjects:
            focus = self.phrase_cleaner(' '.join(dobjects))
        elif subjects:
            focus = self.phrase_cleaner(' '.join(subjects))
        elif pobjects:
            focus = self.phrase_cleaner(' '.join(pobjects))
        if focus in query: focus = None

        subject_with_objects = []
        for dobject in dobjects:
            subject_with_objects.append(dobject)
        for subject in subjects:
            subject_with_objects.append(subject)
        for pobject in pobjects:
            subject_with_objects.append(pobject)
        subject_with_objects = ' '.join(subject_with_objects)

        if query:
            if tts_output: userin.say("Please wait...", True, False)
            wh_question = []
            for word in doc:
				if word.tag_ in ['WDT','WP','WP$','WRB']:
					wh_question.append(word.text.upper())
            page = wikipedia.page(wikipedia.search(query)[0])
            wiki_doc = self.nlp(page.content)
            sentences = [sent.string.strip() for sent in wiki_doc.sents]
            #return [' '.join(subjects),' '.join(pobjects)]
            all_entities = []
            findings = []
            mention = {}
            subject_entities_by_wordnet = None
            if 'WHAT' in wh_question:
                subject_entities_by_wordnet = self.wordnet_entity_determiner(subject_with_objects,tts_output)
            for sentence in reversed(sentences):
                sentence = self.nlp(sentence)
                for ent in sentence.ents:
                    all_entities.append(ent.text)
                    for wh in wh_question:
                        if self.entity_map.has_key(wh.upper()):
                            target_entities = self.entity_map[wh.upper()]
                            if wh.upper() == 'WHAT':
                                target_entities = []
                                for subject_entity_by_wordnet in subject_entities_by_wordnet:
                                    target_entities.append(subject_entity_by_wordnet)
                            if ent.label_ in target_entities:
                                findings.append(ent.text)
                                if focus:
                                    if focus in sentence.text:
                                        mention[ent.text] = 1.0 #* sentence.text.count(focus)
                                    else:
                                        mention[ent.text] = 0.0
                                else:
                                    mention[ent.text] = 0.0

            if findings:
                frequency = collections.Counter(findings)
                max_freq = max(frequency.values())
                for key, value in frequency.iteritems():
                    frequency[key] = float(value) / max_freq

                precedence = {}
                unique = list(set(findings))
                for i in range(len(unique)):
                    precedence[unique[i]] = float(len(unique) - i) / len(unique)

                proximity = {}
                subject_indices = []
                for i in range(len(all_entities)):
                    for subject in subjects:
                        for word in subject.split():
                            if word in all_entities[i]:
                                subject_indices.append(i)
                for i in range(len(all_entities)):
                    for index in subject_indices:
                        inverse_distance = float((len(all_entities) - 1) - abs(i - index)) / (len(all_entities) - 1)
                        if proximity.has_key(all_entities[i]):
                            proximity[all_entities[i]] = (proximity[all_entities[i]] + inverse_distance) / 2
                        else:
                            proximity[all_entities[i]] = inverse_distance
                    if not proximity.has_key(all_entities[i]):
                            proximity[all_entities[i]] = 0

                ranked = {}
                for key, value in frequency.iteritems():
                    if key not in query:
                        ranked[key] = value * self.coefficient['frequency'] + precedence[key] * self.coefficient['precedence'] + proximity[key] * self.coefficient['proximity'] + mention[key] * self.coefficient['mention']
                if not tts_output: print sorted(ranked.items(), key=lambda x:x[1])[::-1][:5]
                if tts_output: userin.say(sorted(ranked.items(), key=lambda x:x[1])[::-1][0][0], True, True)
                return sorted(ranked.items(), key=lambda x:x[1])[::-1][0][0]
            else:
                if not tts_output: print "Sorry, I couldn't find anything worthy to answer your question."
                if tts_output: userin.say("Sorry, I couldn't find anything worthy to answer your question.", True, True)
                return False

    def wordnet_entity_determiner(self,subject,tts_output):
        print subject
        entity_samples_map = {
                'PERSON': ['person','character','human','individual','name'],
                'NORP': ['nationality','religion','politics'],
                'FACILITY': ['building','airport','highway','bridge','port'],
                'ORG': ['company','agency','institution','university'],
                'GPE': ['country','city','state','address','capital'],
                'LOC': ['geography','mountain','ocean','river'],
                'PRODUCT': ['product','object','vehicle','food'],
                'EVENT': ['hurricane','battle','war','sport'],
                'WORK_OF_ART': ['art','book','song','painting'],
                'LANGUAGE': ['language','accent','dialect','speech'],
                'DATE': ['year','month','day'],
                'TIME': ['time','hour','minute'],
                'PERCENT': ['percent','rate','ratio','fee'],
                'MONEY': ['money','cash','salary','wealth'],
                'QUANTITY': ['measurement','amount','distance','height','population'],
                'ORDINAL': ['ordinal','first','second','third'],
                'CARDINAL': ['cardinal','number','amount','mathematics']
        }
        doc = self.nlp(subject.decode('utf-8'))
        subject = []
        for word in doc:
            #if word.pos_ not in ['PUNCT','SYM','X','CONJ','DET','ADP','SPACE']:
            if word.pos_ == 'NOUN':
                subject.append(word.text.lower())
        entity_scores = {}
        for entity, samples in entity_samples_map.iteritems():
            entity_scores[entity] = 0
            for sample in samples:
                sample_wn = wn.synset(sample + '.n.01')
                for word in subject:
                    word_wn = wn.synset(word + '.n.01')
                    entity_scores[entity] += word_wn.path_similarity(sample_wn)
            entity_scores[entity] = entity_scores[entity] / len(samples)
        if not tts_output: print sorted(entity_scores.items(), key=lambda x:x[1])[::-1][:3]
        result = sorted(entity_scores.items(), key=lambda x:x[1])[::-1][0][0]
        if result == 'FACILITY': return [result,'ORG']
        return [result]

    def randomize_coefficients(self):
        coeff1 = round(random.uniform(0.00, 0.98),2)
        coeff2 = round(random.uniform(0.00, (1 - coeff1)),2)
        coeff3 = round(random.uniform(0.00, (1 - (coeff1 + coeff2))),2)
        coeff4 = 1 - (coeff1 + coeff2 + coeff3)
        self.coefficient = {'frequency': coeff1, 'precedence': coeff2, 'proximity': coeff3, 'mention': coeff4}

    def phrase_cleaner(self,phrase):
        clean_phrase = []
        for word in self.nlp(phrase.decode('utf-8')):
            if word.pos_ not in ['PUNCT','SYM','X','CONJ','DET','ADP','SPACE']:<commit_msg>Go to next Wikipedia article if there are no findings<commit_after>from dragonfire.utilities import Data
import wikipedia
import spacy
import collections
from nltk.corpus import wordnet as wn
import random

class Engine():

    def __init__(self):
        self.nlp = spacy.load('en')
        self.entity_map = {
                'WHO': ['PERSON'],
                'WHAT': ['PERSON','NORP','FACILITY','ORG','GPE','LOC','PRODUCT','EVENT','WORK_OF_ART','LANGUAGE','DATE','TIME','PERCENT','MONEY','QUANTITY','ORDINAL','CARDINAL'],
                'WHEN': ['DATE','TIME','EVENT'],
                'WHERE': ['FACILITY','GPE','LOC']
        }
        self.coefficient = {'frequency': 0.27, 'precedence': 0.27, 'proximity': 0.23, 'mention': 0.23}

    def respond(self,com,tts_output=False):
        userin = Data([" "]," ")
        doc = self.nlp(com.decode('utf-8'))
        query = None
        subjects = []
        pobjects = []
        dobjects = []
        for np in doc.noun_chunks:
            #print(np.text, np.root.text, np.root.dep_, np.root.head.text)
            if (np.root.dep_ == 'nsubj' or np.root.dep_ == 'nsubjpass') and np.root.tag_ != 'WP':
                subjects.append(np.text.encode('utf-8'))
            if np.root.dep_ == 'pobj':
                pobjects.append(np.text.encode('utf-8'))
            if np.root.dep_ == 'dobj':
                dobjects.append(np.text.encode('utf-8'))
        if pobjects:
            query = ' '.join(pobjects)
        elif subjects:
            query = ' '.join(subjects)
        elif dobjects:
            query = ' '.join(dobjects)
        else:
            if not tts_output: print "Sorry, I don't understand the subject of your question."
            if tts_output: userin.say("Sorry, I don't understand the subject of your question.")
            return False

        focus = None
        if dobjects:
            focus = self.phrase_cleaner(' '.join(dobjects))
        elif subjects:
            focus = self.phrase_cleaner(' '.join(subjects))
        elif pobjects:
            focus = self.phrase_cleaner(' '.join(pobjects))
        if focus in query: focus = None

        subject_with_objects = []
        for dobject in dobjects:
            subject_with_objects.append(dobject)
        for subject in subjects:
            subject_with_objects.append(subject)
        for pobject in pobjects:
            subject_with_objects.append(pobject)
        subject_with_objects = ' '.join(subject_with_objects)

        if query:
            if tts_output: userin.say("Please wait...", True, False)
            wh_question = []
            for word in doc:
				if word.tag_ in ['WDT','WP','WP$','WRB']:
					wh_question.append(word.text.upper())
            findings = []
            nth_page = 0
            while not findings:
                page = wikipedia.page(wikipedia.search(query)[nth_page])
                nth_page += 1
                if nth_page > 5: break
                wiki_doc = self.nlp(page.content)
                sentences = [sent.string.strip() for sent in wiki_doc.sents]
                #return [' '.join(subjects),' '.join(pobjects)]
                all_entities = []
                mention = {}
                subject_entities_by_wordnet = None
                if 'WHAT' in wh_question:
                    subject_entities_by_wordnet = self.wordnet_entity_determiner(subject_with_objects,tts_output)
                for sentence in reversed(sentences):
                    sentence = self.nlp(sentence)
                    for ent in sentence.ents:
                        all_entities.append(ent.text)
                        for wh in wh_question:
                            if self.entity_map.has_key(wh.upper()):
                                target_entities = self.entity_map[wh.upper()]
                                if wh.upper() == 'WHAT':
                                    target_entities = []
                                    for subject_entity_by_wordnet in subject_entities_by_wordnet:
                                        target_entities.append(subject_entity_by_wordnet)
                                if ent.label_ in target_entities:
                                    findings.append(ent.text)
                                    if focus:
                                        if focus in sentence.text:
                                            mention[ent.text] = 1.0 #* sentence.text.count(focus)
                                        else:
                                            mention[ent.text] = 0.0
                                    else:
                                        mention[ent.text] = 0.0

            if findings:
                frequency = collections.Counter(findings)
                max_freq = max(frequency.values())
                for key, value in frequency.iteritems():
                    frequency[key] = float(value) / max_freq

                precedence = {}
                unique = list(set(findings))
                for i in range(len(unique)):
                    precedence[unique[i]] = float(len(unique) - i) / len(unique)

                proximity = {}
                subject_indices = []
                for i in range(len(all_entities)):
                    for subject in subjects:
                        for word in subject.split():
                            if word in all_entities[i]:
                                subject_indices.append(i)
                for i in range(len(all_entities)):
                    for index in subject_indices:
                        inverse_distance = float((len(all_entities) - 1) - abs(i - index)) / (len(all_entities) - 1)
                        if proximity.has_key(all_entities[i]):
                            proximity[all_entities[i]] = (proximity[all_entities[i]] + inverse_distance) / 2
                        else:
                            proximity[all_entities[i]] = inverse_distance
                    if not proximity.has_key(all_entities[i]):
                            proximity[all_entities[i]] = 0

                ranked = {}
                for key, value in frequency.iteritems():
                    if key not in query:
                        ranked[key] = value * self.coefficient['frequency'] + precedence[key] * self.coefficient['precedence'] + proximity[key] * self.coefficient['proximity'] + mention[key] * self.coefficient['mention']
                if not tts_output: print sorted(ranked.items(), key=lambda x:x[1])[::-1][:5]
                if tts_output: userin.say(sorted(ranked.items(), key=lambda x:x[1])[::-1][0][0], True, True)
                return sorted(ranked.items(), key=lambda x:x[1])[::-1][0][0]
            else:
                if not tts_output: print "Sorry, I couldn't find anything worthy to answer your question."
                if tts_output: userin.say("Sorry, I couldn't find anything worthy to answer your question.", True, True)
                return False

    def wordnet_entity_determiner(self,subject,tts_output):
        print subject
        entity_samples_map = {
                'PERSON': ['person','character','human','individual','name'],
                'NORP': ['nationality','religion','politics'],
                'FACILITY': ['building','airport','highway','bridge','port'],
                'ORG': ['company','agency','institution','university'],
                'GPE': ['country','city','state','address','capital'],
                'LOC': ['geography','mountain','ocean','river'],
                'PRODUCT': ['product','object','vehicle','food'],
                'EVENT': ['hurricane','battle','war','sport'],
                'WORK_OF_ART': ['art','book','song','painting'],
                'LANGUAGE': ['language','accent','dialect','speech'],
                'DATE': ['year','month','day'],
                'TIME': ['time','hour','minute'],
                'PERCENT': ['percent','rate','ratio','fee'],
                'MONEY': ['money','cash','salary','wealth'],
                'QUANTITY': ['measurement','amount','distance','height','population'],
                'ORDINAL': ['ordinal','first','second','third'],
                'CARDINAL': ['cardinal','number','amount','mathematics']
        }
        doc = self.nlp(subject.decode('utf-8'))
        subject = []
        for word in doc:
            #if word.pos_ not in ['PUNCT','SYM','X','CONJ','DET','ADP','SPACE']:
            if word.pos_ == 'NOUN':
                subject.append(word.text.lower())
        entity_scores = {}
        for entity, samples in entity_samples_map.iteritems():
            entity_scores[entity] = 0
            for sample in samples:
                sample_wn = wn.synset(sample + '.n.01')
                for word in subject:
                    word_wn = wn.synset(word + '.n.01')
                    entity_scores[entity] += word_wn.path_similarity(sample_wn)
            entity_scores[entity] = entity_scores[entity] / len(samples)
        if not tts_output: print sorted(entity_scores.items(), key=lambda x:x[1])[::-1][:3]
        result = sorted(entity_scores.items(), key=lambda x:x[1])[::-1][0][0]
        if result == 'FACILITY': return [result,'ORG']
        if result == 'PRODUCT': return [result,'ORG']
        return [result]

    def randomize_coefficients(self):
        coeff1 = round(random.uniform(0.00, 0.98),2)
        coeff2 = round(random.uniform(0.00, (1 - coeff1)),2)
        coeff3 = round(random.uniform(0.00, (1 - (coeff1 + coeff2))),2)
        coeff4 = 1 - (coeff1 + coeff2 + coeff3)
        self.coefficient = {'frequency': coeff1, 'precedence': coeff2, 'proximity': coeff3, 'mention': coeff4}

    def phrase_cleaner(self,phrase):
        clean_phrase = []
        for word in self.nlp(phrase.decode('utf-8')):
            if word.pos_ not in ['PUNCT','SYM','X','CONJ','DET','ADP','SPACE']:
{'fraction_non_alphanumeric': 0.13786317261380704, 'fraction_numerical': 0.008804184577140193, 'mean_word_length': 1.7593598170905973, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 24, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1171331', 'n_tokens_mistral': 5613, 'n_tokens_neox': 5337, 'n_words': 1086}