<commit_before>package internal

import (
	"encoding/json"
	"errors"
	"net/url"
	"strings"

	"github.com/sirupsen/logrus"
)

// PathValidator -- validates Path values
//
//
// - JSON array ()
type PathValidator struct {
	// AllowMultiple -- if true, attempt to json.Unmarshal into a string slice before validating the whole thing
	AllowMultiple bool

	// ValidSchemes -- if not nil, parse as URL and check if the Scheme matches one of the ones defined here (case insensitive -> only store lower case keys in here)
	ValidSchemes map[string]bool
}

// Validate -- checks if the given value meets our criteria
func (v PathValidator) Validate(value string) error {
	var err error
	if v.AllowMultiple {
		var paths []string
		if err = json.Unmarshal([]byte(value), &paths); err == nil {
			// valid JSON list -> check each individual item

			for _, value := range paths {
				if err = v.validatePath(value); err != nil {
					return err
				}
			}
			return nil
		} else if strings.HasPrefix(value, "[") {
			logrus.WithField("path", value).WithError(err).Warn("your path looks like it should be a JSON array - but it's not formatted properly")
		}
	}
	return v.validatePath(value)
}

// validatePath -- checks an individual path
func (v PathValidator) validatePath(value string) error {
	if v.ValidSchemes != nil {
		var u, err = url.Parse(value)
		if err != nil {
			logrus.WithError(err).WithField("url", value).Error("PathValidator: invalid URL")
			return err
		}

		if u.RawQuery != "" {
			logrus.WithField("url", value).Error("URL can't have a query string")
			return errors.New("URL query string unsupported")
		}
		if u.Fragment != "" {
			logrus.WithField("url", value).Error("URL can't have a fragment")
			return errors.New("URL fragment unsupported")
		}

		var scheme = strings.ToLower(u.Scheme)
		if !v.ValidSchemes[scheme] {
			logrus.WithField("url", value).Error("URL scheme not supported")
			return errors.New("unsupported URL scheme: " + u.Scheme)
		}

		switch scheme {
		case "http", "https":
			if u.Path != "" || u.RawPath != "" || u.Opaque != "" {
				logrus.WithField("url", value).Errorf("%s URLs should only have a hostPort, but no path", strings.ToUpper(scheme))
				return errors.New("PathValidator: got path in HTTP(s) URL")
			}
		case "", "file", "unix":
			if u.Host != "" {
				logrus.WithField("url", value).Error("file paths can't have a hostname portion")
				logrus.Info("  you may have used something like file://etc/motd instead of file:/etc/motd (or file:///etc/motd)")
				return errors.New("PathValidator: got host in file URL")
			}
		}

		return nil
	}

	// simple path: assume it's valid
	return nil
}

// Value -- puts the parsed data into a Value
func (v PathValidator) Value(raw string) Value {
	var rc Value

	if len(raw) == 0 {
		return rc // empty value
	}

	if v.AllowMultiple {
		if err := json.Unmarshal([]byte(raw), &rc.values); err == nil {
			return rc // return split data
		}
	}

	// default behaviour: put it into the first slice
	rc.values = []string{raw}
	return rc
}<commit_msg>moved PathValidator.Validate() code into .Value() now (so .Error will always be filled)<commit_after>package internal

import (
	"encoding/json"
	"errors"
	"net/url"
	"strings"

	"github.com/sirupsen/logrus"
)

// PathValidator -- validates Path values
//
//
// - JSON array ()
type PathValidator struct {
	// AllowMultiple -- if true, attempt to json.Unmarshal into a string slice before validating the whole thing
	AllowMultiple bool

	// ValidSchemes -- if not nil, parse as URL and check if the Scheme matches one of the ones defined here (case insensitive -> only store lower case keys in here)
	ValidSchemes map[string]bool
}

// Validate -- checks if the given value meets our criteria
func (v PathValidator) Validate(value string) error {
	return v.Value(value).Error
}

// validatePath -- checks an individual path
func (v PathValidator) validatePath(value string) error {
	if v.ValidSchemes != nil {
		var u, err = url.Parse(value)
		if err != nil {
			logrus.WithError(err).WithField("url", value).Error("PathValidator: invalid URL")
			return err
		}

		if u.RawQuery != "" {
			logrus.WithField("url", value).Error("URL can't have a query string")
			return errors.New("URL query string unsupported")
		}
		if u.Fragment != "" {
			logrus.WithField("url", value).Error("URL can't have a fragment")
			return errors.New("URL fragment unsupported")
		}

		var scheme = strings.ToLower(u.Scheme)
		if !v.ValidSchemes[scheme] {
			logrus.WithField("url", value).Error("URL scheme not supported")
			return errors.New("unsupported URL scheme: " + u.Scheme)
		}

		switch scheme {
		case "http", "https":
			if u.Path != "" || u.RawPath != "" || u.Opaque != "" {
				if u.Path != "/" {
					logrus.WithField("url", value).Errorf("%s URLs should only have a hostPort, but no path", strings.ToUpper(scheme))
					return errors.New("PathValidator: got path in HTTP(s) URL")
				}
			}
		case "", "file", "unix":
			if u.Host != "" {
				logrus.WithField("url", value).Error("file paths can't have a hostname portion")
				logrus.Info("  you may have used something like file://etc/motd instead of file:/etc/motd (or file:///etc/motd)")
				return errors.New("PathValidator: got host in file URL")
			}
		}

		return nil
	}

	// simple path: assume it's valid
	return nil
}

// Value -- puts the parsed data into a Value
func (v PathValidator) Value(raw string) Value {
	if len(raw) == 0 {
		return Value{} // empty value
	}

	if v.AllowMultiple {
		var values []string
		if err := json.Unmarshal([]byte(raw), &values); err == nil {
			// valid JSON list -> check each individual item
			for _, value := range values {
				if err = v.validatePath(value); err != nil {
					break
				}
			}
			return Value{
				values: values,
				Error:  err,
			}
		} else if strings.HasPrefix(raw, "[") {
			logrus.WithField("path", raw).WithError(err).Warn("your path looks like it should be a JSON array - but it's not formatted properly")
		}
	}

	// default behaviour: put it into the first slice
	return Value{
		values: []string{raw},
		Error:  v.validatePath(raw),
	}
}
{'fraction_non_alphanumeric': 0.13459963975765515, 'fraction_numerical': 0.0003274930407728836, 'mean_word_length': 3.353528153955809, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4100471', 'n_tokens_mistral': 2119, 'n_tokens_neox': 1892, 'n_words': 641}