# Path: spark_parser/scanner.py
# class GenericScanner:
#     """A class which can be used subclass off of to make
# 
#     specific sets of scanners. Scanner methods that are subclassed off
#     of this that begin with t_ will be introspected in their
#     documentation string and uses as a regular expression in a token pattern.
#     For example:
# 
#         def t_add_op(self, s):
#         r'[+-]'
#         t = GenericToken(kind='ADD_OP', attr=s)
#         self.rv.append(t)
#     """
#     def __init__(self):
#         pattern = self.reflect()
#         self.pos = 0
#         self.re = re.compile(pattern, re.VERBOSE)
# 
#         self.index2func = {}
#         for name, number in self.re.groupindex.items():
#             self.index2func[number-1] = getattr(self, 't_' + name)
# 
#     def makeRE(self, name):
#         doc = getattr(self, name).__doc__
#         rv = '(?P<%s>%s)' % (name[2:], doc)
#         return rv
# 
#     def reflect(self):
#         rv = []
#         for name in list(_namelist(self)):
#             if name[:2] == 't_' and name != 't_default':
#                 rv.append(self.makeRE(name))
#         rv.append(self.makeRE('t_default'))
#         return '|'.join(rv)
# 
#     def error(self, s):
#         """Simple-minded error handler. see py2_scan for another
#         possibility.'
#         """
#         print("Lexical error in %s at position %s" % (s, self.pos))
#         raise SystemExit
# 
#     def tokenize(self, s):
#         self.pos = 0
#         n = len(s)
#         while self.pos < n:
#             m = self.re.match(s, self.pos)
#             if m is None:
#                 self.error(s)
# 
#             groups = m.groups()
#             for i in range(len(groups)):
#                 if groups[i] and i in self.index2func:
#                     self.index2func[i](groups[i])
#             self.pos = m.end()
# 
#     def t_default(self, s):
#         r'( \n )+'
#         pass


from spark_parser.scanner import GenericScanner
from py2_token import PythonToken
import re
SYMBOL2NAME = {
    '@': 'AT',    '`': 'BACKTICK',
    ':': 'COLON', ',': 'COMMA',
    '.': 'DOT',
    }

ENDMARKER = r''   # ctrl-d

class Python2Scanner(GenericScanner):

    def error(self, s, pos):
        """Show text and a carot under that. For example:
x = 2y + z
     ^
"""
        print("Lexical error:")
        print("%s" % s[:pos+10])  # + 10 for trailing context
        print("%s^" % (" "*(pos-1)))
        for t in self.rv: print(t)
        raise SystemExit

    def __init__(self):
        self.is_newline = True
        self.indents = [0]
        self.lineno = 1
        self.column = 0
        GenericScanner.__init__(self)

    def tokenize(self, string):
        self.rv = []

# Path: spark_parser/scanner.py
# class GenericScanner:
#     """A class which can be used subclass off of to make
# 
#     specific sets of scanners. Scanner methods that are subclassed off
#     of this that begin with t_ will be introspected in their
#     documentation string and uses as a regular expression in a token pattern.
#     For example:
# 
#         def t_add_op(self, s):
#         r'[+-]'
#         t = GenericToken(kind='ADD_OP', attr=s)
#         self.rv.append(t)
#     """
#     def __init__(self):
#         pattern = self.reflect()
#         self.pos = 0
#         self.re = re.compile(pattern, re.VERBOSE)
# 
#         self.index2func = {}
#         for name, number in self.re.groupindex.items():
#             self.index2func[number-1] = getattr(self, 't_' + name)
# 
#     def makeRE(self, name):
#         doc = getattr(self, name).__doc__
#         rv = '(?P<%s>%s)' % (name[2:], doc)
#         return rv
# 
#     def reflect(self):
#         rv = []
#         for name in list(_namelist(self)):
#             if name[:2] == 't_' and name != 't_default':
#                 rv.append(self.makeRE(name))
#         rv.append(self.makeRE('t_default'))
#         return '|'.join(rv)
# 
#     def error(self, s):
#         """Simple-minded error handler. see py2_scan for another
#         possibility.'
#         """
#         print("Lexical error in %s at position %s" % (s, self.pos))
#         raise SystemExit
# 
#     def tokenize(self, s):
#         self.pos = 0
#         n = len(s)
#         while self.pos < n:
#             m = self.re.match(s, self.pos)
#             if m is None:
#                 self.error(s)
# 
#             groups = m.groups()
#             for i in range(len(groups)):
#                 if groups[i] and i in self.index2func:
#                     self.index2func[i](groups[i])
#             self.pos = m.end()
# 
#     def t_default(self, s):
#         r'( \n )+'
#         pass


# Path: example/python2/py2_scan.py
from spark_parser.scanner import GenericScanner
from py2_token import PythonToken
import re

SYMBOL2NAME = {
    '@': 'AT',    '`': 'BACKTICK',
    ':': 'COLON', ',': 'COMMA',
    '.': 'DOT',
    }

ENDMARKER = r''   # ctrl-d

class Python2Scanner(GenericScanner):

    def error(self, s, pos):
        """Show text and a carot under that. For example:
x = 2y + z
     ^
"""
        print("Lexical error:")
        print("%s" % s[:pos+10])  # + 10 for trailing context
        print("%s^" % (" "*(pos-1)))
        for t in self.rv: print(t)
        raise SystemExit

    def __init__(self):
        self.is_newline = True
        self.indents = [0]
        self.lineno = 1
        self.column = 0
        GenericScanner.__init__(self)

    def tokenize(self, string):
        self.rv = []

        GenericScanner.tokenize(self, string)