<commit_before>// Copyright © 2016 <NAME>.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package code

import (
	"encoding/hex"
	"errors"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"

	"github.com/atotto/clipboard"
	"github.com/spf13/cobra"

	"github.com/philpennock/character/resultset"
	"github.com/philpennock/character/sources"
	"github.com/philpennock/character/unicode"

	"github.com/philpennock/character/commands/root"
)

var flags struct {
	base       intconvBase
	clipboard  bool
	netVerbose bool
	livevim    bool
	utf8hex    bool
	verbose    bool
}

// FIXME: make dedicated type, embed search info

// ErrUnknownCodepoint means the specified codepoint is not assigned
var ErrUnknownCodepoint = errors.New("unknown character codepoint")

// When decoding, we may encounter problems
type deferredError struct {
	arg string
	err error
}

var codeCmd = &cobra.Command{
	Use:   "code [codepoint ...]",
	Short: "shows character with codepoint",
	Run: func(cmd *cobra.Command, args []string) {
		srcs := sources.NewFast()
		if flags.verbose && flags.livevim {
			srcs.LoadLiveVim()
		}

		// If given utf8hex flag, then we need to pre-munge the parameters
		deferredErrors := make([]deferredError, 0, 20)

		if flags.utf8hex {
			// WAG the capacity
			newargs := make([]string, 0, len(args)*4)
			matchHexPairSeq := regexp.MustCompile(`^(?:%[0-9A-Fa-f]{2})+`)

			for _, arg := range args {
				// If we have %-encoded, then take non-%-preceded entries as literal character
				// If we don't, then assume that we just have hex strings
				if strings.ContainsRune(arg, '%') {
					for len(arg) > 0 {
						nextPercent := strings.IndexByte(arg, '%')
						if nextPercent < 0 {
							for _, c := range arg {
								newargs = append(newargs, strconv.Itoa(int(c)))
							}
							arg = ""
							continue
						}
						if nextPercent > 0 {
							for _, c := range arg[:nextPercent] {
								newargs = append(newargs, strconv.Itoa(int(c)))
							}
							arg = arg[nextPercent:]
							// do not 'continue', handle inline immediately next
						}
						matches := matchHexPairSeq.FindStringSubmatch(arg)
						if matches == nil {
							deferredErrors = append(deferredErrors, deferredError{arg: arg, err: errors.New("malformed %hex seq")})
							arg = ""
							continue
						}
						got := matches[0]
						arg = arg[len(got):]
						got = strings.Replace(got, "%", "", -1)

						toAdd, defErr := codepointsFromHexString(got)
						if toAdd != nil {
							newargs = append(newargs, toAdd...)
						}
						if defErr != nil {
							deferredErrors = append(deferredErrors, *defErr)
							continue
						}
					}
				} else {
					toAdd, defErr := codepointsFromHexString(arg)
					if toAdd != nil {
						newargs = append(newargs, toAdd...)
					}
					if defErr != nil {
						deferredErrors = append(deferredErrors, *defErr)
						continue
					}<commit_msg>code: handle `=` for hex escape sequence, like `%`<commit_after>// Copyright © 2016 <NAME>.
// All rights reserved, except as granted under license.
// Licensed per file LICENSE.txt

package code

import (
	"encoding/hex"
	"errors"
	"regexp"
	"strconv"
	"strings"
	"unicode/utf8"

	"github.com/atotto/clipboard"
	"github.com/spf13/cobra"

	"github.com/philpennock/character/resultset"
	"github.com/philpennock/character/sources"
	"github.com/philpennock/character/unicode"

	"github.com/philpennock/character/commands/root"
)

var flags struct {
	base       intconvBase
	clipboard  bool
	netVerbose bool
	livevim    bool
	utf8hex    bool
	verbose    bool
}

// FIXME: make dedicated type, embed search info

// ErrUnknownCodepoint means the specified codepoint is not assigned
var ErrUnknownCodepoint = errors.New("unknown character codepoint")

// When decoding, we may encounter problems
type deferredError struct {
	arg string
	err error
}

var codeCmd = &cobra.Command{
	Use:   "code [codepoint ...]",
	Short: "shows character with codepoint",
	Run: func(cmd *cobra.Command, args []string) {
		srcs := sources.NewFast()
		if flags.verbose && flags.livevim {
			srcs.LoadLiveVim()
		}

		// If given utf8hex flag, then we need to pre-munge the parameters
		deferredErrors := make([]deferredError, 0, 20)

		if flags.utf8hex {
			// WAG the capacity
			newargs := make([]string, 0, len(args)*4)
			matchHexPairSeq := regexp.MustCompile(`^(?:[%=][0-9A-Fa-f]{2})+`)

			// If we have %-encoded, then take non-%-preceded entries as literal character
			// If we don't, then assume that we just have hex strings
			// Handle = too, for MIME HDR encoding
			// We handle either, but only one or the other per command-invocation.
			var escapeChar rune = 0

			for _, arg := range args {
				if escapeChar == 0 {
					if strings.ContainsRune(arg, '%') {
						escapeChar = '%'
					} else if strings.ContainsRune(arg, '=') {
						escapeChar = '='
					}
				}

				if escapeChar != 0 && strings.ContainsRune(arg, escapeChar) {
					for len(arg) > 0 {
						nextEscape := strings.IndexRune(arg, escapeChar)
						if nextEscape < 0 {
							for _, c := range arg {
								newargs = append(newargs, strconv.Itoa(int(c)))
							}
							arg = ""
							continue
						}
						if nextEscape > 0 {
							for _, c := range arg[:nextEscape] {
								newargs = append(newargs, strconv.Itoa(int(c)))
							}
							arg = arg[nextEscape:]
							// do not 'continue', handle inline immediately next
						}
						matches := matchHexPairSeq.FindStringSubmatch(arg)
						if matches == nil {
							emsg := "malformed " + string(escapeChar) + "hex sequene"
							deferredErrors = append(deferredErrors, deferredError{arg: arg, err: errors.New(emsg)})
							arg = ""
							continue
						}
						got := matches[0]
						arg = arg[len(got):]
						got = strings.Replace(got, string(escapeChar), "", -1)

						toAdd, defErr := codepointsFromHexString(got)
						if toAdd != nil {
							newargs = append(newargs, toAdd...)
						}
						if defErr != nil {
							deferredErrors = append(deferredErrors, *defErr)
							continue
						}
					}
				} else {
					toAdd, defErr := codepointsFromHexString(arg)
					if toAdd != nil {
						newargs = append(newargs, toAdd...)
					}
					if defErr != nil {
						deferredErrors = append(deferredErrors, *defErr)
						continue
					}
{'fraction_non_alphanumeric': 0.12416534181240063, 'fraction_numerical': 0.0077901430842607314, 'mean_word_length': 2.7335311572700296, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6847821', 'n_tokens_mistral': 2634, 'n_tokens_neox': 2052, 'n_words': 540}