parameters: (lst, defs, ban=[])
docstring: Replace the tokens in lst, using the macros provided in defs, and a list of macros that cannot be re-applied
  
  	:param lst: list of tokens
  	:type lst: list of tuple(token, value)
  	:param defs: macro definitions
  	:type defs: dict
  	:param ban: macros that cannot be substituted (recursion is not allowed)
  	:type ban: list of string
  	:return: the new list of tokens
  	:rtype: value, list
docstring_summary: Replace the tokens in lst, using the macros provided in defs, and a list of macros that cannot be re-applied
docstring_tokens: Replace the tokens in lst using the macros provided in defs and a list of macros that cannot be re - applied
function: def reduce_tokens(lst, defs, ban=[]):
  	"""
  	Replace the tokens in lst, using the macros provided in defs, and a list of macros that cannot be re-applied
  
  	:param lst: list of tokens
  	:type lst: list of tuple(token, value)
  	:param defs: macro definitions
  	:type defs: dict
  	:param ban: macros that cannot be substituted (recursion is not allowed)
  	:type ban: list of string
  	:return: the new list of tokens
  	:rtype: value, list
  	"""
  
  	i = 0
  	while i < len(lst):
  		(p, v) = lst[i]
  
  		if p == IDENT and v == "defined":
  			del lst[i]
  			if i < len(lst):
  				(p2, v2) = lst[i]
  				if p2 == IDENT:
  					if v2 in defs:
  						lst[i] = (NUM, 1)
  					else:
  						lst[i] = (NUM, 0)
  				elif p2 == OP and v2 == '(':
  					del lst[i]
  					(p2, v2) = lst[i]
  					del lst[i] # remove the ident, and change the ) for the value
  					if v2 in defs:
  						lst[i] = (NUM, 1)
  					else:
  						lst[i] = (NUM, 0)
  				else:
  					raise PreprocError('Invalid define expression %r' % lst)
  
  		elif p == IDENT and v in defs:
  
  			if isinstance(defs[v], str):
  				a, b = extract_macro(defs[v])
  				defs[v] = b
  			macro_def = defs[v]
  			to_add = macro_def[1]
  
  			if isinstance(macro_def[0], list):
  				# macro without arguments
  				del lst[i]
  				accu = to_add[:]
  				reduce_tokens(accu, defs, ban+[v])
  				for tmp in accu:
  					lst.insert(i, tmp)
  					i += 1
  			else:
  				# collect the arguments for the funcall
  
  				args = []
  				del lst[i]
  
  				if i >= len(lst):
  					raise PreprocError('expected ( after %r (got nothing)' % v)
  
  				(p2, v2) = lst[i]
  				if p2 != OP or v2 != '(':
  					raise PreprocError('expected ( after %r' % v)
  
  				del lst[i]
  
  				one_param = []
  				count_paren = 0
  				while i < len(lst):
  					p2, v2 = lst[i]
  
  					del lst[i]
  					if p2 == OP and count_paren == 0:
  						if v2 == '(':
  							one_param.append((p2, v2))
  							count_paren += 1
  						elif v2 == ')':
  							if one_param:
  								args.append(one_param)
  							break
  						elif v2 == ',':
  							if not one_param:
  								raise PreprocError('empty param in funcall %r' % v)
  							args.append(one_param)
  							one_param = []
  						else:
  							one_param.append((p2, v2))
  					else:
  						one_param.append((p2, v2))
  						if   v2 == '(':
  							count_paren += 1
  						elif v2 == ')':
  							count_paren -= 1
  				else:
  					raise PreprocError('malformed macro')
  
  				# substitute the arguments within the define expression
  				accu = []
  				arg_table = macro_def[0]
  				j = 0
  				while j < len(to_add):
  					(p2, v2) = to_add[j]
  
  					if p2 == OP and v2 == '#':
  						# stringize is for arguments only
  						if j+1 < len(to_add) and to_add[j+1][0] == IDENT and to_add[j+1][1] in arg_table:
  							toks = args[arg_table[to_add[j+1][1]]]
  							accu.append((STR, stringize(toks)))
  							j += 1
  						else:
  							accu.append((p2, v2))
  					elif p2 == OP and v2 == '##':
  						# token pasting, how can man invent such a complicated system?
  						if accu and j+1 < len(to_add):
  							# we have at least two tokens
  
  							t1 = accu[-1]
  
  							if to_add[j+1][0] == IDENT and to_add[j+1][1] in arg_table:
  								toks = args[arg_table[to_add[j+1][1]]]
  
  								if toks:
  									accu[-1] = paste_tokens(t1, toks[0]) #(IDENT, accu[-1][1] + toks[0][1])
  									accu.extend(toks[1:])
  								else:
  									# error, case "a##"
  									accu.append((p2, v2))
  									accu.extend(toks)
  							elif to_add[j+1][0] == IDENT and to_add[j+1][1] == '__VA_ARGS__':
  								# first collect the tokens
  								va_toks = []
  								st = len(macro_def[0])
  								pt = len(args)
  								for x in args[pt-st+1:]:
  									va_toks.extend(x)
  									va_toks.append((OP, ','))
  								if va_toks:
  									va_toks.pop() # extra comma
  								if len(accu)>1:
  									(p3, v3) = accu[-1]
  									(p4, v4) = accu[-2]
  									if v3 == '##':
  										# remove the token paste
  										accu.pop()
  										if v4 == ',' and pt < st:
  											# remove the comma
  											accu.pop()
  								accu += va_toks
  							else:
  								accu[-1] = paste_tokens(t1, to_add[j+1])
  
  							j += 1
  						else:
  							# Invalid paste, case    "##a" or "b##"
  							accu.append((p2, v2))
  
  					elif p2 == IDENT and v2 in arg_table:
  						toks = args[arg_table[v2]]
  						reduce_tokens(toks, defs, ban+[v])
  						accu.extend(toks)
  					else:
  						accu.append((p2, v2))
  
  					j += 1
  
  
  				reduce_tokens(accu, defs, ban+[v])
  
  				for x in range(len(accu)-1, -1, -1):
  					lst.insert(i, accu[x])
  
  		i += 1
function_tokens: def reduce_tokens ( lst , defs , ban = [ ] ) : i = 0 while i < len ( lst ) : ( p , v ) = lst [ i ] if p == IDENT and v == "defined" : del lst [ i ] if i < len ( lst ) : ( p2 , v2 ) = lst [ i ] if p2 == IDENT : if v2 in defs : lst [ i ] = ( NUM , 1 ) else : lst [ i ] = ( NUM , 0 ) elif p2 == OP and v2 == '(' : del lst [ i ] ( p2 , v2 ) = lst [ i ] del lst [ i ] # remove the ident, and change the ) for the value if v2 in defs : lst [ i ] = ( NUM , 1 ) else : lst [ i ] = ( NUM , 0 ) else : raise PreprocError ( 'Invalid define expression %r' % lst ) elif p == IDENT and v in defs : if isinstance ( defs [ v ] , str ) : a , b = extract_macro ( defs [ v ] ) defs [ v ] = b macro_def = defs [ v ] to_add = macro_def [ 1 ] if isinstance ( macro_def [ 0 ] , list ) : # macro without arguments del lst [ i ] accu = to_add [ : ] reduce_tokens ( accu , defs , ban + [ v ] ) for tmp in accu : lst . insert ( i , tmp ) i += 1 else : # collect the arguments for the funcall args = [ ] del lst [ i ] if i >= len ( lst ) : raise PreprocError ( 'expected ( after %r (got nothing)' % v ) ( p2 , v2 ) = lst [ i ] if p2 != OP or v2 != '(' : raise PreprocError ( 'expected ( after %r' % v ) del lst [ i ] one_param = [ ] count_paren = 0 while i < len ( lst ) : p2 , v2 = lst [ i ] del lst [ i ] if p2 == OP and count_paren == 0 : if v2 == '(' : one_param . append ( ( p2 , v2 ) ) count_paren += 1 elif v2 == ')' : if one_param : args . append ( one_param ) break elif v2 == ',' : if not one_param : raise PreprocError ( 'empty param in funcall %r' % v ) args . append ( one_param ) one_param = [ ] else : one_param . append ( ( p2 , v2 ) ) else : one_param . append ( ( p2 , v2 ) ) if v2 == '(' : count_paren += 1 elif v2 == ')' : count_paren -= 1 else : raise PreprocError ( 'malformed macro' ) # substitute the arguments within the define expression accu = [ ] arg_table = macro_def [ 0 ] j = 0 while j < len ( to_add ) : ( p2 , v2 ) = to_add [ j ] if p2 == OP and v2 == '#' : # stringize is for arguments only if j + 1 < len ( to_add ) and to_add [ j + 1 ] [ 0 ] == IDENT and to_add [ j + 1 ] [ 1 ] in arg_table : toks = args [ arg_table [ to_add [ j + 1 ] [ 1 ] ] ] accu . append ( ( STR , stringize ( toks ) ) ) j += 1 else : accu . append ( ( p2 , v2 ) ) elif p2 == OP and v2 == '##' : # token pasting, how can man invent such a complicated system? if accu and j + 1 < len ( to_add ) : # we have at least two tokens t1 = accu [ - 1 ] if to_add [ j + 1 ] [ 0 ] == IDENT and to_add [ j + 1 ] [ 1 ] in arg_table : toks = args [ arg_table [ to_add [ j + 1 ] [ 1 ] ] ] if toks : accu [ - 1 ] = paste_tokens ( t1 , toks [ 0 ] ) #(IDENT, accu[-1][1] + toks[0][1]) accu . extend ( toks [ 1 : ] ) else : # error, case "a##" accu . append ( ( p2 , v2 ) ) accu . extend ( toks ) elif to_add [ j + 1 ] [ 0 ] == IDENT and to_add [ j + 1 ] [ 1 ] == '__VA_ARGS__' : # first collect the tokens va_toks = [ ] st = len ( macro_def [ 0 ] ) pt = len ( args ) for x in args [ pt - st + 1 : ] : va_toks . extend ( x ) va_toks . append ( ( OP , ',' ) ) if va_toks : va_toks . pop ( ) # extra comma if len ( accu ) > 1 : ( p3 , v3 ) = accu [ - 1 ] ( p4 , v4 ) = accu [ - 2 ] if v3 == '##' : # remove the token paste accu . pop ( ) if v4 == ',' and pt < st : # remove the comma accu . pop ( ) accu += va_toks else : accu [ - 1 ] = paste_tokens ( t1 , to_add [ j + 1 ] ) j += 1 else : # Invalid paste, case    "##a" or "b##" accu . append ( ( p2 , v2 ) ) elif p2 == IDENT and v2 in arg_table : toks = args [ arg_table [ v2 ] ] reduce_tokens ( toks , defs , ban + [ v ] ) accu . extend ( toks ) else : accu . append ( ( p2 , v2 ) ) j += 1 reduce_tokens ( accu , defs , ban + [ v ] ) for x in range ( len ( accu ) - 1 , - 1 , - 1 ) : lst . insert ( i , accu [ x ] ) i += 1