<commit_before>package interpreter

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"../cpath"
)

type StatementT struct {
	Args     []string
	RawArgs  []string
	Redirect []*Redirecter
	Term     string
}

var prefix []string = []string{" 0<", " 1>", " 2>"}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return fmt.Sprintf("%d", LastErrorLevel)
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

func OurGetEnv(name string) (string, bool) {
	value := os.Getenv(name)
	if value != "" {
		return value, true
	} else if m := rxUnicode.FindStringSubmatch(name); m != nil {
		ucode, _ := strconv.ParseInt(m[1], 16, 32)
		return fmt.Sprintf("%c", rune(ucode)), true
	} else if f, ok := PercentFunc[strings.ToUpper(name)]; ok {
		return f(), true
	} else {
		return "", false
	}
}

func chomp(buffer *bytes.Buffer) {
	original := buffer.String()
	buffer.Reset()
	var lastchar rune
	for i, ch := range original {
		if i > 0 {
			buffer.WriteRune(lastchar)
		}
		lastchar = ch
	}
}

const NOTQUOTED = '\000'

const EMPTY_COMMAND_FOUND = "Empty command found"

func string2word(source_ string, removeQuote bool) string {
	var buffer bytes.Buffer
	source := strings.NewReader(source_)

	lastchar := ' '
	quoteNow := NOTQUOTED
	yenCount := 0
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if ch == '~' && unicode.IsSpace(lastchar) {
			if home := cpath.GetHome(); home != "" {
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' && quoteNow != '\'' && yenCount%2 == 0 {
			var nameBuf bytes.Buffer
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break
				}
				if ch == '%' {
					if value, ok := OurGetEnv(nameBuf.String()); ok {
						buffer.WriteString(value)
					} else {
						buffer.WriteRune('%')
						source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					}
					break
				}
				if ch == '=' {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break
				}
				nameBuf.WriteRune(ch)
			}
			continue
		}

		if quoteNow != NOTQUOTED && ch == quoteNow && yenCount%2 == 0 {
			if !removeQuote {
				buffer.WriteRune(ch)
			}
			// Close Quotation.
			for ; yenCount >= 2; yenCount -= 2 {
				buffer.WriteRune('\\')
			}<commit_msg>Fix bug: echo %123456% -> %23456% ('1' is lost)<commit_after>package interpreter

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"../cpath"
)

type StatementT struct {
	Args     []string
	RawArgs  []string
	Redirect []*Redirecter
	Term     string
}

var prefix []string = []string{" 0<", " 1>", " 2>"}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return fmt.Sprintf("%d", LastErrorLevel)
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

func OurGetEnv(name string) (string, bool) {
	value := os.Getenv(name)
	if value != "" {
		return value, true
	} else if m := rxUnicode.FindStringSubmatch(name); m != nil {
		ucode, _ := strconv.ParseInt(m[1], 16, 32)
		return fmt.Sprintf("%c", rune(ucode)), true
	} else if f, ok := PercentFunc[strings.ToUpper(name)]; ok {
		return f(), true
	} else {
		return "", false
	}
}

func chomp(buffer *bytes.Buffer) {
	original := buffer.String()
	buffer.Reset()
	var lastchar rune
	for i, ch := range original {
		if i > 0 {
			buffer.WriteRune(lastchar)
		}
		lastchar = ch
	}
}

const NOTQUOTED = '\000'

const EMPTY_COMMAND_FOUND = "Empty command found"

func string2word(source_ string, removeQuote bool) string {
	var buffer bytes.Buffer
	source := strings.NewReader(source_)

	lastchar := ' '
	quoteNow := NOTQUOTED
	yenCount := 0
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if ch == '~' && unicode.IsSpace(lastchar) {
			if home := cpath.GetHome(); home != "" {
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' && quoteNow != '\'' && yenCount%2 == 0 {
			var nameBuf bytes.Buffer
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break
				}
				if ch == '%' {
					if value, ok := OurGetEnv(nameBuf.String()); ok {
						buffer.WriteString(value)
					} else {
						buffer.WriteRune('%')
						source.Seek(-int64(nameBuf.Len()+1), io.SeekCurrent)
					}
					break
				}
				if ch == '=' {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break
				}
				nameBuf.WriteRune(ch)
			}
			continue
		}

		if quoteNow != NOTQUOTED && ch == quoteNow && yenCount%2 == 0 {
			if !removeQuote {
				buffer.WriteRune(ch)
			}
			// Close Quotation.
			for ; yenCount >= 2; yenCount -= 2 {
				buffer.WriteRune('\\')
			}
{'fraction_non_alphanumeric': 0.17387944358578053, 'fraction_numerical': 0.013330757341576507, 'mean_word_length': 2.8692077727952165, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 12, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3788221', 'n_tokens_mistral': 2323, 'n_tokens_neox': 1961, 'n_words': 441}