<commit_before>import sys
import contextlib
import cStringIO
from random import randint
import collections
import pkg_resources
from lxml import etree
import re
from tinydb import TinyDB, Query
from os.path import expanduser

class Aiml():

	def __init__(self):
		self.replacements = collections.OrderedDict()
		self.replacements["I'M"] = "YOU ARE"
		self.replacements["I WAS"] = "YOU WERE"
		self.replacements["I "] = "YOU "
		self.replacements["MY"] = "YOUR"
		self.grammar_model = collections.OrderedDict()
		self.grammar_model["(?P<subject>.*) (?P<verbtense>IS|ARE|WAS|WERE|WILL BE) (?P<clause>.*)"] = "(?:WHO|WHERE|WHEN|WHY|WHAT|WHICH|HOW)(?:.*)(?P<verbtense>IS|ARE|WAS|WERE|WILL BE) (?P<subject>.*)"
		self.grammar_model["(?P<subject>.*) (?P<verbtense>MEANS?|HAS|HAVE|LIVES?) (?P<clause>.*)"] = "(?:WHO|WHERE|WHEN|WHY|WHAT|WHICH|HOW)(?:.*)(?:DOES|DO) (?P<subject>.*) (?P<verbtense>MEAN|HAVE|LIVE)"
		self.grammar_model["(?P<subject>.*) (?P<verbtense>SAID) (?P<clause>.*)"] = "(?:WHO|WHERE|WHEN|WHY|WHAT|WHICH|HOW)(?:.*)(?:DID) (?P<subject>.*) (?P<verbtense>SAY)"
		home = expanduser("~")
		self.db = TinyDB(home + '/.dragonfire.json')

	def respond(self,com):
		forget = "^(?:FORGET|UPDATE) (?:ABOUT )?(?P<subject>.*)"
		capture = re.search(forget, com)
		if capture:
			if self.db.remove(Query().subject == capture.group('subject')):
				return "OK, I FORGOT EVERYTHING I KNOW ABOUT " + capture.group('subject').upper()
			else:
				return "I WASN'T EVEN KNOW ANYTHING ABOUT " + capture.group('subject').upper()

		for statement,question in self.grammar_model.iteritems():

			capture = re.search(question, com)
			if capture:
				result = self.db.search(Query().subject == capture.group('subject'))
				if result:
					for row in result:
						original_clause = row['clause']
						for item in result[:]:
							if row['verbtense'] == item['verbtense'] and original_clause != item['clause']:
								row['clause'] = row['clause'] + ' AND ' + item['clause']
								result.remove(item)
					answer = ''
					for row in result:<commit_msg>Add a define/explain [this] directive to the Complex Regex Grammar Model<commit_after>import sys
import contextlib
import cStringIO
from random import randint
import collections
import pkg_resources
from lxml import etree
import re
from tinydb import TinyDB, Query
from os.path import expanduser

class Aiml():

	def __init__(self):
		self.replacements = collections.OrderedDict()
		self.replacements["I'M"] = "YOU ARE"
		self.replacements["I WAS"] = "YOU WERE"
		self.replacements["I "] = "YOU "
		self.replacements["MY"] = "YOUR"
		self.grammar_model = collections.OrderedDict()
		self.grammar_model["(?P<subject>.*) (?P<verbtense>IS|ARE|WAS|WERE|WILL BE) (?P<clause>.*)"] = "(?:WHO|WHERE|WHEN|WHY|WHAT|WHICH|HOW)(?:.*)(?P<verbtense>IS|ARE|WAS|WERE|WILL BE) (?P<subject>.*)"
		self.grammar_model["(?P<subject>.*) (?P<verbtense>MEANS?|HAS|HAVE|LIVES?) (?P<clause>.*)"] = "(?:WHO|WHERE|WHEN|WHY|WHAT|WHICH|HOW)(?:.*)(?:DOES|DO) (?P<subject>.*) (?P<verbtense>MEAN|HAVE|LIVE)"
		self.grammar_model["(?P<subject>.*) (?P<verbtense>SAID) (?P<clause>.*)"] = "(?:WHO|WHERE|WHEN|WHY|WHAT|WHICH|HOW)(?:.*)(?:DID) (?P<subject>.*) (?P<verbtense>SAY)"
		self.grammar_model["\b\B"] = "(?:DEFINE|EXPLAIN|TELL ME ABOUT) (?P<subject>.*)"
		home = expanduser("~")
		self.db = TinyDB(home + '/.dragonfire.json')

	def respond(self,com):
		forget = "^(?:FORGET|UPDATE) (?:ABOUT )?(?P<subject>.*)"
		capture = re.search(forget, com)
		if capture:
			if self.db.remove(Query().subject == capture.group('subject')):
				return "OK, I FORGOT EVERYTHING I KNOW ABOUT " + capture.group('subject').upper()
			else:
				return "I WASN'T EVEN KNOW ANYTHING ABOUT " + capture.group('subject').upper()

		for statement,question in self.grammar_model.iteritems():

			capture = re.search(question, com)
			if capture:
				result = self.db.search(Query().subject == capture.group('subject'))
				if result:
					for row in result:
						original_clause = row['clause']
						for item in result[:]:
							if row['verbtense'] == item['verbtense'] and original_clause != item['clause']:
								row['clause'] = row['clause'] + ' AND ' + item['clause']
								result.remove(item)
					answer = ''
					for row in result:
{'fraction_non_alphanumeric': 0.19948002836208933, 'fraction_numerical': 0.0, 'mean_word_length': 5.045714285714285, 'pattern_counts': {'":': 0, '<': 36, '<?xml version=': 0, '>': 36, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2714340', 'n_tokens_mistral': 1813, 'n_tokens_neox': 1561, 'n_words': 302}