func NewDefaultClassifier() *TokenClassifier {
	typeMap := map[int32]RuneTokenType{}
	addRuneClass(&typeMap, RUNE_CHAR, RUNETOKEN_CHAR)
	addRuneClass(&typeMap, RUNE_SPACE, RUNETOKEN_SPACE)
	addRuneClass(&typeMap, RUNE_ESCAPING_QUOTE, RUNETOKEN_ESCAPING_QUOTE)
	addRuneClass(&typeMap, RUNE_NONESCAPING_QUOTE, RUNETOKEN_NONESCAPING_QUOTE)
	addRuneClass(&typeMap, RUNE_ESCAPE, RUNETOKEN_ESCAPE)
	addRuneClass(&typeMap, RUNE_COMMENT, RUNETOKEN_COMMENT)
	return &TokenClassifier{
		typeMap: typeMap}
}
/*
Create a new classifier for basic ASCII characters.
package shlex
import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"strings"
)
type TokenType int
type RuneTokenType int
type lexerState int
type Token struct {
	tokenType TokenType
	value     string
}
func (a *Token) Equal(b *Token) bool {
	if a == nil || b == nil {
		return false
	}
	if a.tokenType != b.tokenType {
		return false
	}
	return a.value == b.value
}
const (
	RUNE_CHAR              string = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789._-,/@$*()+=><:;&^%~|!?[]{}"
	RUNE_SPACE             string = " \t\r\n"
	RUNE_ESCAPING_QUOTE    string = "\""
	RUNE_NONESCAPING_QUOTE string = "'"
	RUNE_ESCAPE                   = "\\"
	RUNE_COMMENT                  = "#"
	RUNETOKEN_UNKNOWN           RuneTokenType = 0
	RUNETOKEN_CHAR              RuneTokenType = 1
	RUNETOKEN_SPACE             RuneTokenType = 2
	RUNETOKEN_ESCAPING_QUOTE    RuneTokenType = 3
	RUNETOKEN_NONESCAPING_QUOTE RuneTokenType = 4
	RUNETOKEN_ESCAPE            RuneTokenType = 5
	RUNETOKEN_COMMENT           RuneTokenType = 6
	RUNETOKEN_EOF               RuneTokenType = 7
	TOKEN_UNKNOWN TokenType = 0
	TOKEN_WORD    TokenType = 1
	TOKEN_SPACE   TokenType = 2
	TOKEN_COMMENT TokenType = 3
	STATE_START           lexerState = 0
	STATE_INWORD          lexerState = 1
	STATE_ESCAPING        lexerState = 2
	STATE_ESCAPING_QUOTED lexerState = 3
	STATE_QUOTED_ESCAPING lexerState = 4
	STATE_QUOTED          lexerState = 5
	STATE_COMMENT         lexerState = 6
	INITIAL_TOKEN_CAPACITY int = 100
)
type TokenClassifier struct {
	typeMap map[int32]RuneTokenType
}
func addRuneClass(typeMap *map[int32]RuneTokenType, runes string, tokenType RuneTokenType) {
	for _, rune := range runes {
		(*typeMap)[int32(rune)] = tokenType
	}
}