func Lexer(s string) ([]Token, error) {
	s = strings.TrimSpace(s)
	var tokens []Token
TokenLoop:
	for i := 0; i < len(s); i++ {
		t := s[i]
		if whitespace.Match([]byte{t}) {
			continue
		}
		for _, seq := range sequenceTypes {
			if match := seq.regex.FindString(s[i:]); match != "" {
				tokens = append(tokens, Token{seq.name, match})
				i += len(match) - 1
				continue TokenLoop
			}
		}
		if name, ok := specialTokens[t]; ok {
			tokens = append(tokens, Token{name, string(t)})
			continue
		}
		return tokens, fmt.Errorf("Unexpected character: %v", string(t))
	}
	tokens = append(tokens, EndOfString)
	return tokens, nil
}
Lexer converts liquid-y strings into lexographic tokens
package liquid
import (
	"fmt"
	"regexp"
	"strings"
)
type Token struct {
	name  string
	value string
}
const (
	tPipe        = "pipe"
	tDot         = "dot"
	tColon       = "colon"
	tComma       = "comma"
	tOpenSquare  = "open_square"
	tCloseSquare = "close_square"
	tOpenRound   = "open_round"
	tCloseRound  = "close_round"
	tQuestion    = "question"
	tDash        = "dash"
	tIdentifier          = "id"
	tSingleStringLiteral = "string"
	tDoubleStringLiteral = "string"
	tNumberLiteral       = "number"
	tDotDot              = "dotdot"
	tComparisonOperator  = "comparison"
	tEndOfString = "end_of_string"
)
var EndOfString = Token{tEndOfString, ""}
var specialTokens = map[uint8]string{
	'|': tPipe,
	'.': tDot,
	':': tColon,
	',': tComma,
	'[': tOpenSquare,
	']': tCloseSquare,
	'(': tOpenRound,
	')': tCloseRound,
	'?': tQuestion,
	'-': tDash,
}
type sequence struct {
	name  string
	regex *regexp.Regexp
}
var sequenceTypes = []sequence{
	{tComparisonOperator, regexp.MustCompile(`^==|!=|<>|<=?|>=?|contains`)},
	{tSingleStringLiteral, regexp.MustCompile(`^'[^\']*'`)},
	{tDoubleStringLiteral, regexp.MustCompile(`^"[^\"]*"`)},
	{tNumberLiteral, regexp.MustCompile(`^-?\d+(\.\d+)?`)},
	{tIdentifier, regexp.MustCompile(`^[a-zA-Z_][\w-]*\??`)},
	{tDotDot, regexp.MustCompile(`^\.\.`)},
}
var whitespace = regexp.MustCompile(`\s`)