<commit_before>package shell

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"github.com/zetamatta/nyagos/nodos"
	"github.com/zetamatta/nyagos/texts"
)

func isSpace(c rune) bool {
	return strings.IndexRune(" \t\n\r\v\f", c) >= 0
}

type StatementT struct {
	Args     []string
	RawArgs  []string
	Redirect []*_Redirecter
	Term     string
}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return fmt.Sprintf("%d", LastErrorLevel)
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

var rxSubstitute = regexp.MustCompile(`^([^\:]+)\:([^\=]+)=(.*)$`)

func ourGetenvSub(name string) (string, bool) {
	m := rxSubstitute.FindStringSubmatch(name)
	if m != nil {
		base, ok := OurGetEnv(m[1])
		if ok {
			return texts.ReplaceIgnoreCase(base, m[2], m[3]), true
		} else {
			return "", false
		}
	} else {
		return OurGetEnv(name)
	}
}

func OurGetEnv(name string) (string, bool) {
	value := os.Getenv(name)
	if value != "" {
		return value, true
	} else if m := rxUnicode.FindStringSubmatch(name); m != nil {
		ucode, _ := strconv.ParseInt(m[1], 16, 32)
		return fmt.Sprintf("%c", rune(ucode)), true
	} else if f, ok := PercentFunc[strings.ToUpper(name)]; ok {
		return f(), true
	} else {
		return "", false
	}
}

func chomp(buffer *bytes.Buffer) {
	buffer.Truncate(buffer.Len() - 1)
}

const NOTQUOTED = '\000'

const EMPTY_COMMAND_FOUND = "Empty command found"

var TildeExpansion = true

func string2word(source_ string, removeQuote bool) string {
	var buffer strings.Builder
	source := strings.NewReader(source_)

	lastchar := ' '
	quoteNow := NOTQUOTED
	yenCount := 0
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if TildeExpansion && ch == '~' && isSpace(lastchar) && quoteNow == NOTQUOTED {
			var name strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					break
				} else if ch == '"' {
					if quoteNow == NOTQUOTED {
						quoteNow = ch
					} else {
						quoteNow = NOTQUOTED
					}
				} else if !unicode.IsLetter(ch) {
					source.UnreadRune()
					break
				}
				name.WriteRune(ch)
			}
			nameStr := strings.Replace(name.String(), `"`, ``, -1)
			if len(nameStr) > 0 {
				u, err := user.Lookup(nameStr)
				if err == nil {
					buffer.WriteString(u.HomeDir)
					lastchar = rune(u.HomeDir[len(u.HomeDir)-1])
				} else {
					buffer.WriteRune('~')
					buffer.WriteString(nameStr)
					lastchar = rune(nameStr[len(nameStr)-1])
				}
				continue
			}
			if home := nodos.GetHome(); home != "" {
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' && quoteNow != '\'' {
			for ; yenCount > 0; yenCount-- {
				buffer.WriteRune('\\')
			}
			var nameBuf strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break<commit_msg>Fix: on`~"\Program Files"`, the first quotation disappeared and `Files` was not contained in the argument<commit_after>package shell

import (
	"bytes"
	"errors"
	"fmt"
	"io"
	"os"
	"os/user"
	"regexp"
	"strconv"
	"strings"
	"unicode"

	"github.com/zetamatta/nyagos/nodos"
	"github.com/zetamatta/nyagos/texts"
)

func isSpace(c rune) bool {
	return strings.IndexRune(" \t\n\r\v\f", c) >= 0
}

type StatementT struct {
	Args     []string
	RawArgs  []string
	Redirect []*_Redirecter
	Term     string
}

var PercentFunc = map[string]func() string{
	"CD": func() string {
		wd, err := os.Getwd()
		if err == nil {
			return wd
		} else {
			return ""
		}
	},
	"ERRORLEVEL": func() string {
		return fmt.Sprintf("%d", LastErrorLevel)
	},
}

var rxUnicode = regexp.MustCompile("^[uU]\\+?([0-9a-fA-F]+)$")

var rxSubstitute = regexp.MustCompile(`^([^\:]+)\:([^\=]+)=(.*)$`)

func ourGetenvSub(name string) (string, bool) {
	m := rxSubstitute.FindStringSubmatch(name)
	if m != nil {
		base, ok := OurGetEnv(m[1])
		if ok {
			return texts.ReplaceIgnoreCase(base, m[2], m[3]), true
		} else {
			return "", false
		}
	} else {
		return OurGetEnv(name)
	}
}

func OurGetEnv(name string) (string, bool) {
	value := os.Getenv(name)
	if value != "" {
		return value, true
	} else if m := rxUnicode.FindStringSubmatch(name); m != nil {
		ucode, _ := strconv.ParseInt(m[1], 16, 32)
		return fmt.Sprintf("%c", rune(ucode)), true
	} else if f, ok := PercentFunc[strings.ToUpper(name)]; ok {
		return f(), true
	} else {
		return "", false
	}
}

func chomp(buffer *bytes.Buffer) {
	buffer.Truncate(buffer.Len() - 1)
}

const NOTQUOTED = '\000'

const EMPTY_COMMAND_FOUND = "Empty command found"

var TildeExpansion = true

func string2word(source_ string, removeQuote bool) string {
	var buffer strings.Builder
	source := strings.NewReader(source_)

	lastchar := ' '
	quoteNow := NOTQUOTED
	yenCount := 0
	for {
		ch, _, err := source.ReadRune()
		if err != nil {
			break
		}
		if TildeExpansion && ch == '~' && isSpace(lastchar) && quoteNow == NOTQUOTED {
			var name strings.Builder
			var undo strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					undo.WriteRune(ch)
					break
				} else if ch == '"' {
					undo.WriteByte('"')
					if quoteNow == NOTQUOTED {
						quoteNow = ch
					} else {
						quoteNow = NOTQUOTED
					}
				} else if !unicode.IsLetter(ch) {
					source.UnreadRune()
					break
				} else {
					undo.WriteRune(ch)
				}
				name.WriteRune(ch)
			}
			nameStr := strings.Replace(name.String(), `"`, ``, -1)
			if len(nameStr) > 0 {
				u, err := user.Lookup(nameStr)
				if err == nil {
					buffer.WriteString(u.HomeDir)
					lastchar = rune(u.HomeDir[len(u.HomeDir)-1])
				} else {
					buffer.WriteByte('~')
					undoStr := undo.String()
					buffer.WriteString(undoStr)
					lastchar = rune(nameStr[len(undoStr)-1])
				}
				continue
			}
			if home := nodos.GetHome(); home != "" {
				if strings.Count(undo.String(), `"`)%2 != 0 {
					buffer.WriteByte('"')
				}
				buffer.WriteString(home)
			} else {
				buffer.WriteRune('~')
			}
			lastchar = '~'
			continue
		}
		if ch == '%' && quoteNow != '\'' {
			for ; yenCount > 0; yenCount-- {
				buffer.WriteRune('\\')
			}
			var nameBuf strings.Builder
			for {
				ch, _, err = source.ReadRune()
				if err != nil {
					buffer.WriteRune('%')
					source.Seek(-int64(nameBuf.Len()), io.SeekCurrent)
					break
{'fraction_non_alphanumeric': 0.17043184263101274, 'fraction_numerical': 0.007684032580298141, 'mean_word_length': 2.9634591961023142, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6669457', 'n_tokens_mistral': 2911, 'n_tokens_neox': 2473, 'n_words': 535}