<commit_before>// Package gcfg reads "gitconfig-like" text-based configuration files with
// "name=value" pairs grouped into sections (gcfg files). Support for modifying
// and/or exporting such files may be added later.
//
// This package is a work in progress, and both the supported file format and
// the API is subject to change.
//
// The syntax is based on that used by git config:
// http://git-scm.com/docs/git-config#_syntax .
// Note that the gcfg syntax may diverge from that of git config in the future
// to a limited degree. Current differences (apart from TODOs listed below) are:
//  - gcfg files must use UTF-8 encoding (for now)
//  - include is not supported (and not planned) 
//
// The package may be usable for handling some of the various "INI file" formats
// used by some programs and libraries, but achieving or maintaining
// compatibility with any of those is not a primary concern.
//
// TODO: besides more docs and tests, add support for:
//  - pointer fields
//  - subsections
//  - multi-value variables (+ internal representation)
//  - returning error context (+ numeric error codes ?)
//  - multiple readers (strings, files)
//  - escaping in strings and long(er) lines (?) (+ regexp-free parser)
//  - modifying files
//  - exporting files (+ metadata handling) (?)
//  - declare encoding (?)
//
package gcfg

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"regexp"
	"strings"
)

var (
	reCmnt    = regexp.MustCompile(`^([^;#"]*)[;#].*$`)
	reCmntQ   = regexp.MustCompile(`^([^;#"]*"[^"]*"[^;#"]*)[;#].*$`)
	reBlank   = regexp.MustCompile(`^\s*$`)
	reSect    = regexp.MustCompile(`^\s*\[(.*)\]\s*$`)
	reVar     = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*([^"\s]*)\s*$`)
	reVarQ    = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*"([^"\n\\]*)"\s*$`)
	reVarDflt = regexp.MustCompile(`^\s*\b(.*)\b\s*$`)
)

const (
	// Default value in case a value for a variable isn't provided.
	DefaultValue = "true"
)

type Bool bool

var boolValues = map[string]interface{}{
	"true": true, "yes": true, "on": true, "1": true,
	"false": false, "no": false, "off": false, "0": false}

func scan(state fmt.ScanState, values map[string]interface{}) (interface{}, error) {
	var rd []rune
	var r rune
	var err error
	for r, _, err = state.ReadRune(); err == nil; r, _, err = state.ReadRune() {
		rd = append(rd, r)
		part := false
		for s, v := range values {
			if strings.EqualFold(string(rd), s) {
				return v, err
			}
			if len(rd) < len(s) && strings.EqualFold(string(rd), s[:len(rd)]) {
				part = true
			}
		}
		if part == false {
			state.UnreadRune()
			return nil, errors.New("unsupported value " + string(rd))
		}
	}
	return nil, err
}

func (b *Bool) Scan(state fmt.ScanState, verb rune) error {
	v, err := scan(state, boolValues)
	switch bb := v.(type) {
	case bool:
		*b = Bool(bb)
		return err
	}
	return err
}

func unref(v reflect.Value) reflect.Value {
	for v.Type().Kind() == reflect.Ptr {
		v = v.Elem()
	}
	return v
}

func fieldFold(v reflect.Value, name string) reflect.Value {
	n := strings.Replace(name, "-", "_", -1)
	return v.FieldByNameFunc(func(fieldName string) bool {
		return strings.EqualFold(n, fieldName)
	})
}

func set(cfg interface{}, sect, name, value string) error {
	vDest := unref(reflect.ValueOf(cfg))
	vSect := unref(fieldFold(vDest, sect))
	vName := unref(fieldFold(vSect, name))
	vAddr := vName.Addr().Interface()
	switch v := vAddr.(type) {
	case *string:
		*v = value
		return nil
	case *bool:
		vAddr = (*Bool)(v)
	}
	var r rune // attempt to read an extra rune
	n, err := fmt.Sscanf(value, "%v%c", vAddr, &r)
	if n < 1 && err != io.EOF {
		return err
	} else if n != 1 || err != io.EOF {
		return fmt.Errorf("failed to parse %q as %#v", value, vName.Type())
	}
	return nil
}

// Parse reads gcfg formatted data from reader and sets the values into the
// corresponding fields in config. Config must be a pointer to a struct.  
func Parse(config interface{}, reader io.Reader) error {
	r := bufio.NewReader(reader)
	sect := (*string)(nil)
	for line := 1; true; line++ {
		l, pre, err := r.ReadLine()
		if err != nil && err != io.EOF {
			return err
		} else if pre {
			return errors.New("line too long")
		}
		// exclude comments
		if c := reCmnt.FindSubmatch(l); c != nil {
			l = c[1]
		} else if c := reCmntQ.FindSubmatch(l); c != nil {
			l = c[1]
		}
		if !reBlank.Match(l) {
			// "switch" based on line contents
			if sec := reSect.FindSubmatch(l); sec != nil {
				strsec := string(sec[1])
				sect = &strsec
			} else if v, vq, vd := reVar.FindSubmatch(l),
				reVarQ.FindSubmatch(l), reVarDflt.FindSubmatch(l); //
			v != nil || vq != nil || vd != nil {
				if sect == nil {<commit_msg>improve parse error messages<commit_after>// Package gcfg reads "gitconfig-like" text-based configuration files with
// "name=value" pairs grouped into sections (gcfg files). Support for modifying
// and/or exporting such files may be added later.
//
// This package is a work in progress, and both the supported file format and
// the API is subject to change.
//
// The syntax is based on that used by git config:
// http://git-scm.com/docs/git-config#_syntax .
// Note that the gcfg syntax may diverge from that of git config in the future
// to a limited degree. Current differences (apart from TODOs listed below) are:
//  - gcfg files must use UTF-8 encoding (for now)
//  - include is not supported (and not planned) 
//
// The package may be usable for handling some of the various "INI file" formats
// used by some programs and libraries, but achieving or maintaining
// compatibility with any of those is not a primary concern.
//
// TODO: besides more docs and tests, add support for:
//  - pointer fields
//  - subsections
//  - multi-value variables (+ internal representation)
//  - returning error context (+ numeric error codes ?)
//  - multiple readers (strings, files)
//  - escaping in strings and long(er) lines (?) (+ regexp-free parser)
//  - modifying files
//  - exporting files (+ metadata handling) (?)
//  - declare encoding (?)
//
package gcfg

import (
	"bufio"
	"errors"
	"fmt"
	"io"
	"os"
	"reflect"
	"regexp"
	"strings"
)

var (
	reCmnt    = regexp.MustCompile(`^([^;#"]*)[;#].*$`)
	reCmntQ   = regexp.MustCompile(`^([^;#"]*"[^"]*"[^;#"]*)[;#].*$`)
	reBlank   = regexp.MustCompile(`^\s*$`)
	reSect    = regexp.MustCompile(`^\s*\[(.*)\]\s*$`)
	reVar     = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*([^"\s]*)\s*$`)
	reVarQ    = regexp.MustCompile(`^\s*([^"=\s]+)\s*=\s*"([^"\n\\]*)"\s*$`)
	reVarDflt = regexp.MustCompile(`^\s*\b(.*)\b\s*$`)
)

const (
	// Default value in case a value for a variable isn't provided.
	DefaultValue = "true"
)

type Bool bool

var boolValues = map[string]interface{}{
	"true": true, "yes": true, "on": true, "1": true,
	"false": false, "no": false, "off": false, "0": false}

func scan(state fmt.ScanState, values map[string]interface{}) (interface{}, error) {
	var rd []rune
	var r rune
	var err error
	for r, _, err = state.ReadRune(); err == nil; r, _, err = state.ReadRune() {
		rd = append(rd, r)
		part := false
		for s, v := range values {
			if strings.EqualFold(string(rd), s) {
				return v, err
			}
			if len(rd) < len(s) && strings.EqualFold(string(rd), s[:len(rd)]) {
				part = true
			}
		}
		if part == false {
			state.UnreadRune()
			return nil, errors.New("unsupported value " + string(rd))
		}
	}
	return nil, err
}

func (b *Bool) Scan(state fmt.ScanState, verb rune) error {
	v, err := scan(state, boolValues)
	switch bb := v.(type) {
	case bool:
		*b = Bool(bb)
		return err
	}
	return err
}

func unref(v reflect.Value) reflect.Value {
	for v.Type().Kind() == reflect.Ptr {
		v = v.Elem()
	}
	return v
}

func fieldFold(v reflect.Value, name string) reflect.Value {
	n := strings.Replace(name, "-", "_", -1)
	return v.FieldByNameFunc(func(fieldName string) bool {
		return strings.EqualFold(n, fieldName)
	})
}

func set(cfg interface{}, sect, name, value string) error {
	vDest := unref(reflect.ValueOf(cfg))
	vSect := unref(fieldFold(vDest, sect))
	vName := unref(fieldFold(vSect, name))
	vAddr := vName.Addr().Interface()
	switch v := vAddr.(type) {
	case *string:
		*v = value
		return nil
	case *bool:
		vAddr = (*Bool)(v)
	}
	// attempt to read an extra rune to make sure the value is consumed 
	var r rune
	n, err := fmt.Sscanf(value, "%v%c", vAddr, &r)
	switch {
	case n < 1 || n == 1 && err != io.EOF:
		return fmt.Errorf("failed to parse %q as %#v: parse error %v", value,
			vName.Type(), err)
	case n > 1:
		return fmt.Errorf("failed to parse %q as %#v: extra characters", value,
			vName.Type())
	case n == 1 && err == io.EOF:
		return nil
	}
	panic("never reached")
}

// Parse reads gcfg formatted data from reader and sets the values into the
// corresponding fields in config. Config must be a pointer to a struct.  
func Parse(config interface{}, reader io.Reader) error {
	r := bufio.NewReader(reader)
	sect := (*string)(nil)
	for line := 1; true; line++ {
		l, pre, err := r.ReadLine()
		if err != nil && err != io.EOF {
			return err
		} else if pre {
			return errors.New("line too long")
		}
		// exclude comments
		if c := reCmnt.FindSubmatch(l); c != nil {
			l = c[1]
		} else if c := reCmntQ.FindSubmatch(l); c != nil {
			l = c[1]
		}
		if !reBlank.Match(l) {
			// "switch" based on line contents
			if sec := reSect.FindSubmatch(l); sec != nil {
				strsec := string(sec[1])
				sect = &strsec
			} else if v, vq, vd := reVar.FindSubmatch(l),
				reVarQ.FindSubmatch(l), reVarDflt.FindSubmatch(l); //
			v != nil || vq != nil || vd != nil {
				if sect == nil {
{'fraction_non_alphanumeric': 0.17185045554508324, 'fraction_numerical': 0.0023039061681851504, 'mean_word_length': 3.278673835125448, 'pattern_counts': {'":': 16, '<': 7, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1097026', 'n_tokens_mistral': 3627, 'n_tokens_neox': 3408, 'n_words': 1082}