parameters: (infofile, allowed_groups, allowed_maps, name_val_pairs)
return_statement: return out
docstring: Generic lexer function for info.txt and src/build-data files
docstring_summary: Generic lexer function for info.txt and src/build-data files
docstring_tokens: Generic lexer function for info . txt and src / build - data files
function: def lex_me_harder(infofile, allowed_groups, allowed_maps, name_val_pairs):
      """
      Generic lexer function for info.txt and src/build-data files
      """
      out = LexResult()
  
      # Format as a nameable Python variable
      def py_var(group):
          return group.replace(':', '_')
  
      lexer = shlex.shlex(open(infofile), infofile, posix=True)
      lexer.wordchars += '=:.<>/,-!?+*' # handle various funky chars in info.txt
  
      groups = allowed_groups + allowed_maps
      for group in groups:
          out.__dict__[py_var(group)] = []
      for (key, val) in name_val_pairs.items():
          out.__dict__[key] = val
  
      def lexed_tokens(): # Convert to an iterator
          while True:
              token = lexer.get_token()
              if token != lexer.eof:
                  yield token
              else:
                  return
  
      for token in lexed_tokens():
          match = re.match('<(.*)>', token)
  
          # Check for a grouping
          if match is not None:
              group = match.group(1)
  
              if group not in groups:
                  raise LexerError('Unknown group "%s"' % (group),
                                   infofile, lexer.lineno)
  
              end_marker = '</' + group + '>'
  
              token = lexer.get_token()
              while token != end_marker:
                  out.__dict__[py_var(group)].append(token)
                  token = lexer.get_token()
                  if token is None:
                      raise LexerError('Group "%s" not terminated' % (group),
                                       infofile, lexer.lineno)
  
          elif token in name_val_pairs.keys():
              if isinstance(out.__dict__[token], list):
                  out.__dict__[token].append(lexer.get_token())
              else:
                  out.__dict__[token] = lexer.get_token()
  
          else: # No match -> error
              raise LexerError('Bad token "%s"' % (token), infofile, lexer.lineno)
  
      for group in allowed_maps:
          out.__dict__[group] = parse_lex_dict(out.__dict__[group], group, infofile)
  
      return out
function_tokens: def lex_me_harder ( infofile , allowed_groups , allowed_maps , name_val_pairs ) : out = LexResult ( ) # Format as a nameable Python variable def py_var ( group ) : return group . replace ( ':' , '_' ) lexer = shlex . shlex ( open ( infofile ) , infofile , posix = True ) lexer . wordchars += '=:.<>/,-!?+*' # handle various funky chars in info.txt groups = allowed_groups + allowed_maps for group in groups : out . __dict__ [ py_var ( group ) ] = [ ] for ( key , val ) in name_val_pairs . items ( ) : out . __dict__ [ key ] = val def lexed_tokens ( ) : # Convert to an iterator while True : token = lexer . get_token ( ) if token != lexer . eof : yield token else : return for token in lexed_tokens ( ) : match = re . match ( '<(.*)>' , token ) # Check for a grouping if match is not None : group = match . group ( 1 ) if group not in groups : raise LexerError ( 'Unknown group "%s"' % ( group ) , infofile , lexer . lineno ) end_marker = '</' + group + '>' token = lexer . get_token ( ) while token != end_marker : out . __dict__ [ py_var ( group ) ] . append ( token ) token = lexer . get_token ( ) if token is None : raise LexerError ( 'Group "%s" not terminated' % ( group ) , infofile , lexer . lineno ) elif token in name_val_pairs . keys ( ) : if isinstance ( out . __dict__ [ token ] , list ) : out . __dict__ [ token ] . append ( lexer . get_token ( ) ) else : out . __dict__ [ token ] = lexer . get_token ( ) else : # No match -> error raise LexerError ( 'Bad token "%s"' % ( token ) , infofile , lexer . lineno ) for group in allowed_maps : out . __dict__ [ group ] = parse_lex_dict ( out . __dict__ [ group ] , group , infofile ) return out