<commit_before>from sympy.logic.boolalg import to_cnf
#from sympy.abc import A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z

import sqlparse
from sqlparse import sql,tokens

def where_to_cnf(where_token):
	#clause = sql.TokenList(where_token.tokens[1:-1])
	symbols = ('A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z')
	comparisons = {}
	trans_form = ''
	sym_idx = 0
	tok_idx = 0
	subtokens = list(where.flatten())
	#print tokens[0].ttype
	#if(subtokens[tok_idx].ttype == tokens.Token.Operator.Comparison):
		#print 'einai comparison'
	#else:
		#print 'den einai comparison'
	n=len(subtokens)
	while (tok_idx < n and sym_idx < 24):
	
		#print tok_idx, ' ' ,subtokens[tok_idx].ttype,' ',subtokens[tok_idx].normalized,' ' 
		if subtokens[tok_idx].ttype in  tokens.Token.Literal: print 'einai kai literal'
		
		#parenthesis handling
		if(subtokens[tok_idx].ttype == tokens.Token.Punctuation and subtokens[tok_idx].value in '()'):
			trans_form += subtokens[tok_idx].value
		#logical operators handling
		elif(subtokens[tok_idx].ttype == tokens.Token.Keyword and subtokens[tok_idx].normalized in ('NOT','AND','OR')):
			if(subtokens[tok_idx].normalized == 'NOT'):
				trans_form += '~'
			elif(subtokens[tok_idx].normalized == 'AND'):
				trans_form += '&'
			else:
				trans_form += '|'
			
		#comparisons handling
		elif(subtokens[tok_idx].ttype == tokens.Token.Operator.Comparison):
			comparison_group = ()
			#print comparison_group
			left = subtokens[:tok_idx]
			left = left[::-1]
			#print left ,'\n reversed left: ',left[::-1]
			
			for i in range(len(left)):
				if(left[i].ttype == tokens.Token.Name and left[i+1].value =='.' and left[i+2].ttype == tokens.Token.Name):
					comparison_group += (left[i],left[i+1],left[i+2])
					break;
				elif(left[i].ttype in tokens.Token.Literal):
					comparison_group += (left[i],)
					break;
			comparison_group = comparison_group[::-1]
			comparison_group += (subtokens[tok_idx],)		
			right = subtokens[tok_idx+1:]
			#print right,'\n'
			
			for i in range(len(right)):
				if(i == len(right)-1):
					comparison_group += (right[i],)
					break;
				else:
					if(right[i].ttype == tokens.Token.Name and right[i+1].value =='.' and right[i+2].ttype == tokens.Token.Name):
						comparison_group += (right[i],right[i+1],right[i+2]) 
						break;
					elif(right[i].ttype in tokens.Token.Literal):
						comparison_group += (right[i],)
						break;
			print " ".join(str(comp) for comp in comparison_group)
			comparisons[symbols[sym_idx]] = comparison_group
			trans_form += symbols[sym_idx]
			sym_idx +=1
		
		tok_idx += 1
	print 'arxiki formula',trans_form 
	print 'cnf : ', to_cnf(trans_form)	
	
#--------------------------debugging-----------------------
sql_test = 'select v1.name as nom,V2.code,count(V2.id) from view1 as v1, view2 as V2 where (v1.code="12345" and v1.id<>2.9) or not( v1.id!=v2.id and v2.stuff=0) group by nom;'
#sql_test='select * from v1 where (v1.id!=123 or v1.id=v2.name) and not(v1.name="kati" or v1.code=g.kati);'
parsed=sqlparse.parse(sql_test)
for where in parsed[0].tokens:
	if(isinstance(where,sql.Where)):
		where_to_cnf(where)<commit_msg>regex for converting formulas back to symbolic logic<commit_after>from sympy.logic.boolalg import to_cnf
#from sympy.abc import A,B,C,D,E,F,G,H,I,J,K,L,M,N,O,P,Q,R,S,T,U,V,W,X,Y,Z

import sqlparse
from sqlparse import sql,tokens

import re

def where_to_cnf(where_token):
	#clause = sql.TokenList(where_token.tokens[1:-1])
	symbols = ('A','B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R','S','T','U','V','W','X','Y','Z')
	comparisons = {}
	trans_form = ''
	sym_idx = 0
	tok_idx = 0
	subtokens = list(where.flatten())
	#print tokens[0].ttype
	#if(subtokens[tok_idx].ttype == tokens.Token.Operator.Comparison):
		#print 'einai comparison'
	#else:
		#print 'den einai comparison'
	n=len(subtokens)
	while (tok_idx < n and sym_idx < 24):
	
		#print tok_idx, ' ' ,subtokens[tok_idx].ttype,' ',subtokens[tok_idx].normalized,' ' 
		if subtokens[tok_idx].ttype in  tokens.Token.Literal: print 'einai kai literal'
		
		#parenthesis handling
		if(subtokens[tok_idx].ttype == tokens.Token.Punctuation and subtokens[tok_idx].value in '()'):
			trans_form += subtokens[tok_idx].value
		#logical operators handling
		elif(subtokens[tok_idx].ttype == tokens.Token.Keyword and subtokens[tok_idx].normalized in ('NOT','AND','OR') or subtokens[tok_idx].value == 'not' ):
			if(subtokens[tok_idx].normalized == 'OR'):
				trans_form += '|'
			elif(subtokens[tok_idx].normalized == 'AND'):
				trans_form += '&'
			else:
				trans_form += '~'
			
		#comparisons handling
		elif(subtokens[tok_idx].ttype == tokens.Token.Operator.Comparison):
			comparison_group = ()
			#print comparison_group
			left = subtokens[:tok_idx]
			left = left[::-1]
			#print left ,'\n reversed left: ',left[::-1]
			
			for i in range(len(left)):
				if(left[i].ttype == tokens.Token.Name and left[i+1].value =='.' and left[i+2].ttype == tokens.Token.Name):
					comparison_group += (left[i],left[i+1],left[i+2])
					break;
				elif(left[i].ttype in tokens.Token.Literal):
					comparison_group += (left[i],)
					break;
			comparison_group = comparison_group[::-1]
			comparison_group += (subtokens[tok_idx],)		
			right = subtokens[tok_idx+1:]
			#print right,'\n'
			
			for i in range(len(right)):
				if(i == len(right)-1):
					comparison_group += (right[i],)
					break;
				else:
					if(right[i].ttype == tokens.Token.Name and right[i+1].value =='.' and right[i+2].ttype == tokens.Token.Name):
						comparison_group += (right[i],right[i+1],right[i+2]) 
						break;
					elif(right[i].ttype in tokens.Token.Literal):
						comparison_group += (right[i],)
						break;
			print " ".join(str(comp) for comp in comparison_group)
			comparisons[symbols[sym_idx]] = comparison_group
			trans_form += symbols[sym_idx]
			sym_idx +=1
		
		tok_idx += 1
	print 'arxiki formula',trans_form , 'cnf : ',to_cnf(trans_form).__str__()
	print 'kati' ,sympylogic_to_symbolic(to_cnf(trans_form).__str__())	



def sympylogic_to_symbolic(sympy_logic_str):
	predicates = re.findall(r'\([A-Z], [A-Z]\)|~\([A-Z], [A-Z]\)', sympy_logic_str)
	symbolic = ''
	print 'sympylogic'
	if len(predicates) == 1:
		symbols =  re.findall(r'[A-Z]', predicates[0])
		for x in range(len(symbols)):
			if(x == (len(symbols)-1)):
				symbolic += symbols[x]
			else:
				symbolic += symbols[x]+' & '
	else:
		for i in range(len(predicates)):
			symbols =  re.findall(r'~[A-Z]|[A-Z]', predicates[i])
			symbolic += '('
			for x in range(len(symbols)):
				if(x == (len(symbols)-1)):
					symbolic += symbols[x]
				else:
					symbolic += symbols[x]+' | '
			if(i == (len(predicates)-1)):
				symbolic += ')'
			else:
				symbolic += ')&'
	return symbolic

	
#--------------------------debugging-----------------------
sql_test = 'select v1.name as nom,V2.code,count(V2.id) from view1 as v1, view2 as V2 where (v1.code="12345" and v1.id<>2.9) or not( v1.id!=v2.id and v2.stuff=0) group by nom;'
#sql_test='select * from v1 where (v1.id!=123 or v1.id=v2.name) and not(v1.name="kati" or v1.code=g.kati);'
parsed=sqlparse.parse(sql_test)
for where in parsed[0].tokens:
	if(isinstance(where,sql.Where)):
		where_to_cnf(where)
{'fraction_non_alphanumeric': 0.200110329609709, 'fraction_numerical': 0.015032409322852021, 'mean_word_length': 4.0819901892081285, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '331264', 'n_tokens_mistral': 3169, 'n_tokens_neox': 2812, 'n_words': 505}