<commit_before>import copy
from . import qdsl

class QueryBody(object):
	def __init__(self):
		self._filter = { 'must':[], 'should':[], 'must_not':[], 'and':[], 'not':[], 'or':[] }
		self._query = { 'must':[], 'should':[], 'must_not':[] }
		self._bool = 'must'
		self._last_part = '_query'
		self._explicit = None

	def chain( self, newpart, **kwargs ):

		condition_map = {'must':'and', 'should':'or', 'must_not': 'not' }
		condition_rev_map = {'and':'must', 'or':'should', 'not': 'must_not' }
		# figure out if we're a filter or a query being chained

		# explicitly stated
		if kwargs.get('type'):
			self._last_part = '_' + kwargs.get('type')
		elif isinstance(newpart, dict):
			_chained = False
			# comes in from dict, recursively chain each part
			if 'filter' in newpart:
				kwargs['type'] = 'filter'
				self.chain(newpart['filter'], **kwargs)
				_chained = True

			if 'query' in newpart:
				kwargs['type'] = 'query'
				self.chain(newpart['query'], **kwargs)
				_chained = True

			# we should have completed the chain at this point so just return
			if _chained:
				return self
		# else uses _last_par and assumes query if not specified
		
		# Explicit conditions only apply to filters
		if self._last_part == '_filter':
			# figure out if we have a top level condition (and, or, not)
			# check if its explicitly stated
			if kwargs.get('with_explicit'):
				self._explicit = kwargs.get('with_explicit')
			# Check if the condition parameter is really a top level
			elif kwargs.get('condition') in ['and','or','not']:
				self._explicit = kwargs.get('condition')

				# reverse the regular condition, this allows for one set of code down below (even if it gets converted back)
				kwargs['condition'] = condition_rev_map[kwargs['condition']]

			# check the part for explicit conditions
			elif isinstance(newpart, dict):
				_chained = False
				for t in ['and','or','not']:
					if t in newpart:
						kwargs['with_explicit'] = t
						self.chain( newpart, **kwargs )
						_chained = True

				# chained above should complete
				if _chained:
					return self
			# else there are no new explicit conditions. If _explicit is set we
			# will use the last value

			# go ahead and parse the newpart. This way when adding to the explicit conditions
			# vs non-explicit we won't have to parse it twice
			# existing conditions in the dictionary will just get parsed recursively
			_condition = 'must'
			if kwargs.get('condition') in ['must','should','must_not']:
				_condition = kwargs.get('condition')
			elif isinstance(newpart, dict):
				_chained = False
				for lt in ['must','should','must_not']:
					kwargs['condition'] = lt
					self.chain( newpart[lt], **kwargs)
					_chained = True
				if _chained:
					return self
			# else treat the condition as 'must'

			if self._explicit:
				# check to see if we need to move existing bools inside an explicit condition
				for btype, ttype in condition_map.iteritems():
					if self._filter.get(btype):
						self._filter[ttype].extend( self._filter[btype] )

						del self._filter[btype]
				
				# we definitely have explicit conditions. Turn the condition into the explicit
				self._explicit = _condition = condition_map[_condition]
				self[_condition].append(newpart)
			else:
				# chain the newpart to the toplevel bool
				if _condition not in self._filter:
					self._filter[_condition] = []
<commit_msg>fixed _filter not being correctly referenced<commit_after>import copy
from . import qdsl

class QueryBody(object):
	def __init__(self):
		self._filter = { 'must':[], 'should':[], 'must_not':[], 'and':[], 'not':[], 'or':[] }
		self._query = { 'must':[], 'should':[], 'must_not':[] }
		self._bool = 'must'
		self._last_part = '_query'
		self._explicit = None

	def chain( self, newpart, **kwargs ):

		condition_map = {'must':'and', 'should':'or', 'must_not': 'not' }
		condition_rev_map = {'and':'must', 'or':'should', 'not': 'must_not' }
		# figure out if we're a filter or a query being chained

		# explicitly stated
		if kwargs.get('type'):
			self._last_part = '_' + kwargs.get('type')
		elif isinstance(newpart, dict):
			_chained = False
			# comes in from dict, recursively chain each part
			if 'filter' in newpart:
				kwargs['type'] = 'filter'
				self.chain(newpart['filter'], **kwargs)
				_chained = True

			if 'query' in newpart:
				kwargs['type'] = 'query'
				self.chain(newpart['query'], **kwargs)
				_chained = True

			# we should have completed the chain at this point so just return
			if _chained:
				return self
		# else uses _last_par and assumes query if not specified
		
		# Explicit conditions only apply to filters
		if self._last_part == '_filter':
			# figure out if we have a top level condition (and, or, not)
			# check if its explicitly stated
			if kwargs.get('with_explicit'):
				self._explicit = kwargs.get('with_explicit')
			# Check if the condition parameter is really a top level
			elif kwargs.get('condition') in ['and','or','not']:
				self._explicit = kwargs.get('condition')

				# reverse the regular condition, this allows for one set of code down below (even if it gets converted back)
				kwargs['condition'] = condition_rev_map[kwargs['condition']]

			# check the part for explicit conditions
			elif isinstance(newpart, dict):
				_chained = False
				for t in ['and','or','not']:
					if t in newpart:
						kwargs['with_explicit'] = t
						self.chain( newpart, **kwargs )
						_chained = True

				# chained above should complete
				if _chained:
					return self
			# else there are no new explicit conditions. If _explicit is set we
			# will use the last value

			# go ahead and parse the newpart. This way when adding to the explicit conditions
			# vs non-explicit we won't have to parse it twice
			# existing conditions in the dictionary will just get parsed recursively
			_condition = 'must'
			if kwargs.get('condition') in ['must','should','must_not']:
				_condition = kwargs.get('condition')
			elif isinstance(newpart, dict):
				_chained = False
				for lt in ['must','should','must_not']:
					kwargs['condition'] = lt
					self.chain( newpart[lt], **kwargs)
					_chained = True
				if _chained:
					return self
			# else treat the condition as 'must'

			if self._explicit:
				# check to see if we need to move existing bools inside an explicit condition
				for btype, ttype in condition_map.iteritems():
					if self._filter.get(btype):
						self._filter[ttype].extend( self._filter[btype] )

						del self._filter[btype]
				
				# we definitely have explicit conditions. Turn the condition into the explicit
				self._explicit = _condition = condition_map[_condition]
				self._filter[_condition].append(newpart)
			else:
				# chain the newpart to the toplevel bool
				if _condition not in self._filter:
					self._filter[_condition] = []

{'fraction_non_alphanumeric': 0.12036358305233837, 'fraction_numerical': 0.0, 'mean_word_length': 3.4911125740618827, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6175778', 'n_tokens_mistral': 2468, 'n_tokens_neox': 2048, 'n_words': 719}