func (sc *scanner) nextToken(val *tokenValue) Token {
start:
	var c rune
	blank := false
	savedLineStart := sc.lineStart
	if sc.lineStart {
		sc.lineStart = false
		col := 0
		for {
			c = sc.peekRune()
			if c == ' ' {
				col++
				sc.readRune()
			} else if c == '\t' {
				const tab = 8
				col += int(tab - (sc.pos.Col-1)%tab)
				sc.readRune()
			} else {
				break
			}
		}
		if c == '#' || c == '\n' || c == 0 {
			blank = true
		}
		if !blank && sc.depth == 0 {
			cur := sc.indentstk[len(sc.indentstk)-1]
			if col > cur {
				sc.dents++
				sc.indentstk = append(sc.indentstk, col)
			} else if col < cur {
				for len(sc.indentstk) > 0 && col < sc.indentstk[len(sc.indentstk)-1] {
					sc.dents--
					sc.indentstk = sc.indentstk[:len(sc.indentstk)-1]  
				}
				if col != sc.indentstk[len(sc.indentstk)-1] {
					sc.error(sc.pos, "unindent does not match any outer indentation level")
				}
			}
		}
	}
	if sc.dents != 0 {
		sc.startToken(val)
		sc.endToken(val)
		if sc.dents < 0 {
			sc.dents++
			return OUTDENT
		} else {
			sc.dents--
			return INDENT
		}
	}
	c = sc.peekRune()
	for c == ' ' || c == '\t' {
		sc.readRune()
		c = sc.peekRune()
	}
	if c == '#' {
		if sc.keepComments {
			sc.startToken(val)
		}
		for c != 0 && c != '\n' {
			sc.readRune()
			c = sc.peekRune()
		}
		if sc.keepComments {
			sc.endToken(val)
			if blank {
				sc.lineComments = append(sc.lineComments, Comment{val.pos, val.raw})
			} else {
				sc.suffixComments = append(sc.suffixComments, Comment{val.pos, val.raw})
			}
		}
	}
	if c == '\n' {
		sc.lineStart = true
		if sc.depth > 0 {
			sc.readRune()
			goto start
		}
		if blank {
			if sc.readline == nil {
				sc.readRune()
				goto start
			} else if len(sc.indentstk) > 1 {
				sc.dents = 1 - len(sc.indentstk)
				sc.indentstk = sc.indentstk[:1]
				goto start
			}
		}
		sc.startToken(val)
		sc.readRune()
		val.raw = "\n"
		return NEWLINE
	}
	if c == 0 {
		if len(sc.indentstk) > 1 {
			if savedLineStart {
				sc.dents = 1 - len(sc.indentstk)
				sc.indentstk = sc.indentstk[:1]
				goto start
			} else {
				sc.lineStart = true
				sc.startToken(val)
				val.raw = "\n"
				return NEWLINE
			}
		}
		sc.startToken(val)
		sc.endToken(val)
		return EOF
	}
	if c == '\\' {
		sc.readRune()
		if sc.peekRune() != '\n' {
			sc.errorf(sc.pos, "stray backslash in program")
		}
		sc.readRune()
		goto start
	}
	sc.startToken(val)
	if c == ',' {
		sc.readRune()
		sc.endToken(val)
		return COMMA
	}
	if c == '"' || c == '\'' {
		return sc.scanString(val, c)
	}
	if isIdentStart(c) {
		if (c == 'r' || c == 'b') && len(sc.rest) > 1 && (sc.rest[1] == '"' || sc.rest[1] == '\'') {
			sc.readRune()
			c = sc.peekRune()
			return sc.scanString(val, c)
		} else if c == 'r' && len(sc.rest) > 2 && sc.rest[1] == 'b' && (sc.rest[2] == '"' || sc.rest[2] == '\'') {
			sc.readRune()
			sc.readRune()
			c = sc.peekRune()
			return sc.scanString(val, c)
		}
		for isIdent(c) {
			sc.readRune()
			c = sc.peekRune()
		}
		sc.endToken(val)
		if k, ok := keywordToken[val.raw]; ok {
			return k
		}
		return IDENT
	}
	switch c {
	case '[', '(', '{':
		sc.depth++
		sc.readRune()
		sc.endToken(val)
		switch c {
		case '[':
			return LBRACK
		case '(':
			return LPAREN
		case '{':
			return LBRACE
		}
		panic("unreachable")
	case ']', ')', '}':
		if sc.depth == 0 {
			sc.errorf(sc.pos, "unexpected %q", c)
		} else {
			sc.depth--
		}
		sc.readRune()
		sc.endToken(val)
		switch c {
		case ']':
			return RBRACK
		case ')':
			return RPAREN
		case '}':
			return RBRACE
		}
		panic("unreachable")
	}
	if isdigit(c) || c == '.' {
		return sc.scanNumber(val, c)
	}
	defer sc.endToken(val)
	switch c {
	case '=', '<', '>', '!', '+', '-', '%', '/', '&', '|', '^':  
		start := sc.pos
		sc.readRune()
		if sc.peekRune() == '=' {
			sc.readRune()
			switch c {
			case '<':
				return LE
			case '>':
				return GE
			case '=':
				return EQL
			case '!':
				return NEQ
			case '+':
				return PLUS_EQ
			case '-':
				return MINUS_EQ
			case '/':
				return SLASH_EQ
			case '%':
				return PERCENT_EQ
			case '&':
				return AMP_EQ
			case '|':
				return PIPE_EQ
			case '^':
				return CIRCUMFLEX_EQ
			}
		}
		switch c {
		case '=':
			return EQ
		case '<':
			if sc.peekRune() == '<' {
				sc.readRune()
				if sc.peekRune() == '=' {
					sc.readRune()
					return LTLT_EQ
				} else {
					return LTLT
				}
			}
			return LT
		case '>':
			if sc.peekRune() == '>' {
				sc.readRune()
				if sc.peekRune() == '=' {
					sc.readRune()
					return GTGT_EQ
				} else {
					return GTGT
				}
			}
			return GT
		case '!':
			sc.error(start, "unexpected input character '!'")
		case '+':
			return PLUS
		case '-':
			return MINUS
		case '/':
			if sc.peekRune() == '/' {
				sc.readRune()
				if sc.peekRune() == '=' {
					sc.readRune()
					return SLASHSLASH_EQ
				} else {
					return SLASHSLASH
				}
			}
			return SLASH
		case '%':
			return PERCENT
		case '&':
			return AMP
		case '|':
			return PIPE
		case '^':
			return CIRCUMFLEX
		}
		panic("unreachable")
	case ':', ';', '~':  
		sc.readRune()
		switch c {
		case ':':
			return COLON
		case ';':
			return SEMI
		case '~':
			return TILDE
		}
		panic("unreachable")
	case '*':  
		sc.readRune()
		switch sc.peekRune() {
		case '*':
			sc.readRune()
			return STARSTAR
		case '=':
			sc.readRune()
			return STAR_EQ
		}
		return STAR
	}
	sc.errorf(sc.pos, "unexpected input character %#q", c)
	panic("unreachable")
}
nextToken is called by the parser to obtain the next input token.
It returns the token value and sets val to the data associated with
the token.

For all our input tokens, the associated data is val.pos (the
position where the token begins), val.raw (the input string
corresponding to the token).  For string and int tokens, the string
and int fields additionally contain the token's interpreted value.
package syntax
import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"math/big"
	"os"
	"strconv"
	"strings"
	"unicode"
	"unicode/utf8"
)
type Token int8
const (
	ILLEGAL Token = iota
	EOF
	NEWLINE
	INDENT
	OUTDENT
	IDENT   
	INT     
	FLOAT   
	STRING  
	BYTES   
	PLUS           
	MINUS          
	STAR           
	SLASH          
	SLASHSLASH     
	PERCENT        
	AMP            
	PIPE           
	CIRCUMFLEX     
	LTLT           
	GTGT           
	TILDE          
	DOT            
	COMMA          
	EQ             
	SEMI           
	COLON          
	LPAREN         
	RPAREN         
	LBRACK         
	RBRACK         
	LBRACE         
	RBRACE         
	LT             
	GT             
	GE             
	LE             
	EQL            
	NEQ            
	PLUS_EQ        
	MINUS_EQ       
	STAR_EQ        
	SLASH_EQ       
	SLASHSLASH_EQ  
	PERCENT_EQ     
	AMP_EQ         
	PIPE_EQ        
	CIRCUMFLEX_EQ  
	LTLT_EQ        
	GTGT_EQ        
	STARSTAR       
	AND
	BREAK
	CONTINUE
	DEF
	ELIF
	ELSE
	FOR
	IF
	IN
	LAMBDA
	LOAD
	NOT
	NOT_IN  
	OR
	PASS
	RETURN
	WHILE
	maxToken
)
func (tok Token) String() string { return tokenNames[tok] }
func (tok Token) GoString() string {
	if tok >= PLUS && tok <= STARSTAR {
		return "'" + tokenNames[tok] + "'"
	}
	return tokenNames[tok]
}
var tokenNames = [...]string{
	ILLEGAL:       "illegal token",
	EOF:           "end of file",
	NEWLINE:       "newline",
	INDENT:        "indent",
	OUTDENT:       "outdent",
	IDENT:         "identifier",
	INT:           "int literal",
	FLOAT:         "float literal",
	STRING:        "string literal",
	PLUS:          "+",
	MINUS:         "-",
	STAR:          "*",
	SLASH:         "/",
	SLASHSLASH:    "//",
	PERCENT:       "%",
	AMP:           "&",
	PIPE:          "|",
	CIRCUMFLEX:    "^",
	LTLT:          "<<",
	GTGT:          ">>",
	TILDE:         "~",
	DOT:           ".",
	COMMA:         ",",
	EQ:            "=",
	SEMI:          ";",
	COLON:         ":",
	LPAREN:        "(",
	RPAREN:        ")",
	LBRACK:        "[",
	RBRACK:        "]",
	LBRACE:        "{",
	RBRACE:        "}",
	LT:            "<",
	GT:            ">",
	GE:            ">=",
	LE:            "<=",
	EQL:           "==",
	NEQ:           "!=",
	PLUS_EQ:       "+=",
	MINUS_EQ:      "-=",
	STAR_EQ:       "*=",
	SLASH_EQ:      "/=",
	SLASHSLASH_EQ: "//=",
	PERCENT_EQ:    "%=",
	AMP_EQ:        "&=",
	PIPE_EQ:       "|=",
	CIRCUMFLEX_EQ: "^=",
	LTLT_EQ:       "<<=",
	GTGT_EQ:       ">>=",
	STARSTAR:      "**",
	AND:           "and",
	BREAK:         "break",
	CONTINUE:      "continue",
	DEF:           "def",
	ELIF:          "elif",
	ELSE:          "else",
	FOR:           "for",
	IF:            "if",
	IN:            "in",
	LAMBDA:        "lambda",
	LOAD:          "load",
	NOT:           "not",
	NOT_IN:        "not in",
	OR:            "or",
	PASS:          "pass",
	RETURN:        "return",
	WHILE:         "while",
}
type FilePortion struct {
	Content             []byte
	FirstLine, FirstCol int32
}
type Position struct {
	file *string  
	Line int32    
	Col  int32    
}
func (p Position) IsValid() bool { return p.file != nil }
func (p Position) Filename() string {
	if p.file != nil {
		return *p.file
	}
	return "<invalid>"
}
func MakePosition(file *string, line, col int32) Position { return Position{file, line, col} }
func (p Position) add(s string) Position {
	if n := strings.Count(s, "\n"); n > 0 {
		p.Line += int32(n)
		s = s[strings.LastIndex(s, "\n")+1:]
		p.Col = 1
	}
	p.Col += int32(utf8.RuneCountInString(s))
	return p
}
func (p Position) String() string {
	file := p.Filename()
	if p.Line > 0 {
		if p.Col > 0 {
			return fmt.Sprintf("%s:%d:%d", file, p.Line, p.Col)
		}
		return fmt.Sprintf("%s:%d", file, p.Line)
	}
	return file
}
func (p Position) isBefore(q Position) bool {
	if p.Line != q.Line {
		return p.Line < q.Line
	}
	return p.Col < q.Col
}
type scanner struct {
	rest           []byte     
	token          []byte     
	pos            Position   
	depth          int        
	indentstk      []int      
	dents          int        
	lineStart      bool       
	keepComments   bool       
	lineComments   []Comment  
	suffixComments []Comment  
	readline func() ([]byte, error)  
}
func newScanner(filename string, src interface{}, keepComments bool) (*scanner, error) {
	var firstLine, firstCol int32 = 1, 1
	if portion, ok := src.(FilePortion); ok {
		firstLine, firstCol = portion.FirstLine, portion.FirstCol
	}
	sc := &scanner{
		pos:          MakePosition(&filename, firstLine, firstCol),
		indentstk:    make([]int, 1, 10),  
		lineStart:    true,
		keepComments: keepComments,
	}
	sc.readline, _ = src.(func() ([]byte, error))  
	if sc.readline == nil {
		data, err := readSource(filename, src)
		if err != nil {
			return nil, err
		}
		sc.rest = data
	}
	return sc, nil
}
func readSource(filename string, src interface{}) ([]byte, error) {
	switch src := src.(type) {
	case string:
		return []byte(src), nil
	case []byte:
		return src, nil
	case io.Reader:
		data, err := ioutil.ReadAll(src)
		if err != nil {
			err = &os.PathError{Op: "read", Path: filename, Err: err}
			return nil, err
		}
		return data, nil
	case FilePortion:
		return src.Content, nil
	case nil:
		return ioutil.ReadFile(filename)
	default:
		return nil, fmt.Errorf("invalid source: %T", src)
	}
}
type Error struct {
	Pos Position
	Msg string
}
func (e Error) Error() string { return e.Pos.String() + ": " + e.Msg }
func (sc *scanner) error(pos Position, s string) {
	panic(Error{pos, s})
}
func (sc *scanner) errorf(pos Position, format string, args ...interface{}) {
	sc.error(pos, fmt.Sprintf(format, args...))
}
func (sc *scanner) recover(err *error) {
	switch e := recover().(type) {
	case nil:
	case Error:
		*err = e
	default:
		*err = Error{sc.pos, fmt.Sprintf("internal error: %v", e)}
		if debug {
			log.Fatal(*err)
		}
	}
}
func (sc *scanner) eof() bool {
	return len(sc.rest) == 0 && !sc.readLine()
}
func (sc *scanner) readLine() bool {
	if sc.readline != nil {
		var err error
		sc.rest, err = sc.readline()
		if err != nil {
			sc.errorf(sc.pos, "%v", err)  
		}
		return len(sc.rest) > 0
	}
	return false
}
func (sc *scanner) peekRune() rune {
	if sc.eof() {
		return 0
	}
	if b := sc.rest[0]; b < utf8.RuneSelf {
		if b == '\r' {
			return '\n'
		}
		return rune(b)
	}
	r, _ := utf8.DecodeRune(sc.rest)
	return r
}
func (sc *scanner) readRune() rune {
	if len(sc.rest) == 0 {
		if !sc.readLine() {
			sc.error(sc.pos, "internal scanner error: readRune at EOF")
		}
		if len(sc.rest) == 0 {
			return 0
		}
	}
	if b := sc.rest[0]; b < utf8.RuneSelf {
		r := rune(b)
		sc.rest = sc.rest[1:]
		if r == '\r' {
			if len(sc.rest) > 0 && sc.rest[0] == '\n' {
				sc.rest = sc.rest[1:]
			}
			r = '\n'
		}
		if r == '\n' {
			sc.pos.Line++
			sc.pos.Col = 1
		} else {
			sc.pos.Col++
		}
		return r
	}
	r, size := utf8.DecodeRune(sc.rest)
	sc.rest = sc.rest[size:]
	sc.pos.Col++
	return r
}
type tokenValue struct {
	raw    string    
	int    int64     
	bigInt *big.Int  
	float  float64   
	string string    
	pos    Position  
}
func (sc *scanner) startToken(val *tokenValue) {
	sc.token = sc.rest
	val.raw = ""
	val.pos = sc.pos
}
func (sc *scanner) endToken(val *tokenValue) {
	if val.raw == "" {
		val.raw = string(sc.token[:len(sc.token)-len(sc.rest)])
	}
}