<commit_before>package utils

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/hex"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
	gofig "github.com/akutz/gofig/types"
	"github.com/akutz/goof"
	"github.com/akutz/gotil"

	"github.com/codedellemc/libstorage/api/context"
	"github.com/codedellemc/libstorage/api/types"
)

var knownHostRX = regexp.MustCompile(`(?i)^([^\s]+?)\s([^\s]+?)\s(.+)$`)

// ParseKnownHost parses a known host line that's in the expected format:
// "host algorithm fingerprint".
func ParseKnownHost(
	ctx types.Context,
	text string) (*types.TLSKnownHost, error) {

	m := knownHostRX.FindStringSubmatch(text)
	if len(m) == 0 {
		return nil, nil
	}

	ctx.WithFields(map[string]interface{}{
		"khHost": m[1],
		"khAlg":  m[2],
		"khSig":  m[3],
	}).Debug("parsing known_hosts file fields")

	buf, err := hex.DecodeString(strings.Replace(m[3], ":", "", -1))
	if err != nil {
		return nil, goof.WithError(
			"error decoding known host fingerprint", err)
	}
	return &types.TLSKnownHost{
		Host:        m[1],
		Alg:         m[2],
		Fingerprint: buf,
	}, nil
}

// ParseTLSConfig returns a new TLS configuration.
func ParseTLSConfig(
	ctx types.Context,
	config gofig.Config,
	fields log.Fields,
	roots ...string) (tlsConfig *types.TLSConfig, tlsErr error) {

	ctx.Debug("parsing tls config")

	pathConfig := context.MustPathConfig(ctx)

	f := func(k string, v interface{}) {
		if fields == nil {
			return
		}
		fields[k] = v
		ctx.WithField(k, v).Debug("tls field set")
	}

	newTLS := func(k string, v interface{}) {
		if tlsConfig != nil {
			return
		}
		ctx.WithField(k, v).Info("tls enabled")
		tlsConfig = &types.TLSConfig{Config: tls.Config{}}
	}

	// check to see if TLS is disabled
	if ok := getBool(config, types.ConfigTLSDisabled, roots...); ok {
		f(types.ConfigTLSDisabled, true)
		ctx.WithField(types.ConfigTLSDisabled, false).Info("tls disabled")
		return nil, nil
	}

	// check to see if TLS is enabled with a simple truthy value
	if ok := getBool(config, types.ConfigTLS, roots...); ok {
		f(types.ConfigTLS, true)
		newTLS(types.ConfigTLS, true)

	} else if v := getString(config, types.ConfigTLS, roots...); v != "" {
		// check to see if TLS is disabled
		if strings.EqualFold(v, "false") {
			f(types.ConfigTLS, "false")
			ctx.WithField(types.ConfigTLS, "false").Info("tls disabled")
			return nil, nil
		}

		// check to see if TLS is enabled with insecure
		if strings.EqualFold(v, "insecure") {
			f(types.ConfigTLS, "insecure")
			newTLS(types.ConfigTLS, "insecure")
			tlsConfig.InsecureSkipVerify = true

			// check to see if TLS is enabled with peers
		} else if strings.EqualFold(v, "verifyPeers") {
			f(types.ConfigTLS, "verifyPeers")
			newTLS(types.ConfigTLS, "verifyPeers")
			tlsConfig.InsecureSkipVerify = true
			tlsConfig.VerifyPeers = true

			// check to see if TLS is enabled with an expected sha256
			// fingerprint
		} else if kh, err := ParseKnownHost(ctx, v); err != nil {
			ctx.Error(err)
			return nil, err
		} else if kh != nil {
			f(types.ConfigTLS, kh.String())
			newTLS(types.ConfigTLS, kh.String())
			tlsConfig.InsecureSkipVerify = true
			tlsConfig.VerifyPeers = true
			tlsConfig.KnownHost = kh
		}
	}

	// always check for the user's known_hosts file
	if gotil.FileExists(pathConfig.UserDefaultTLSKnownHosts) {
		newTLS("usrKnownHosts", pathConfig.UserDefaultTLSKnownHosts)
		f("usrKnownHosts", pathConfig.UserDefaultTLSKnownHosts)
		tlsConfig.UsrKnownHosts = pathConfig.UserDefaultTLSKnownHosts
		tlsConfig.VerifyPeers = true
	}

	if getBool(config, types.ConfigTLSInsecure, roots...) {
		newTLS(types.ConfigTLSInsecure, true)
		f(types.ConfigTLSInsecure, true)
		tlsConfig.InsecureSkipVerify = true
	}
<commit_msg>TLS Known Hosts Fix<commit_after>package utils

import (
	"crypto/tls"
	"crypto/x509"
	"encoding/hex"
	"io/ioutil"
	"os"
	"regexp"
	"strings"

	log "github.com/Sirupsen/logrus"
	gofig "github.com/akutz/gofig/types"
	"github.com/akutz/goof"
	"github.com/akutz/gotil"

	"github.com/codedellemc/libstorage/api/context"
	"github.com/codedellemc/libstorage/api/types"
)

var knownHostRX = regexp.MustCompile(`(?i)^([^\s]+?)\s([^\s]+?)\s(.+)$`)

// ParseKnownHost parses a known host line that's in the expected format:
// "host algorithm fingerprint".
func ParseKnownHost(
	ctx types.Context,
	text string) (*types.TLSKnownHost, error) {

	m := knownHostRX.FindStringSubmatch(text)
	if len(m) == 0 {
		return nil, nil
	}

	ctx.WithFields(map[string]interface{}{
		"khHost": m[1],
		"khAlg":  m[2],
		"khSig":  m[3],
	}).Debug("parsing known_hosts file fields")

	buf, err := hex.DecodeString(strings.Replace(m[3], ":", "", -1))
	if err != nil {
		return nil, goof.WithError(
			"error decoding known host fingerprint", err)
	}
	return &types.TLSKnownHost{
		Host:        m[1],
		Alg:         m[2],
		Fingerprint: buf,
	}, nil
}

// ParseTLSConfig returns a new TLS configuration.
func ParseTLSConfig(
	ctx types.Context,
	config gofig.Config,
	fields log.Fields,
	roots ...string) (tlsConfig *types.TLSConfig, tlsErr error) {

	ctx.Debug("parsing tls config")

	pathConfig := context.MustPathConfig(ctx)

	f := func(k string, v interface{}) {
		if fields == nil {
			return
		}
		fields[k] = v
		ctx.WithField(k, v).Debug("tls field set")
	}

	newTLS := func(k string, v interface{}) {
		if tlsConfig != nil {
			return
		}
		ctx.WithField(k, v).Info("tls enabled")
		tlsConfig = &types.TLSConfig{Config: tls.Config{}}
	}

	// check to see if TLS is disabled
	if ok := getBool(config, types.ConfigTLSDisabled, roots...); ok {
		f(types.ConfigTLSDisabled, true)
		ctx.WithField(types.ConfigTLSDisabled, false).Info("tls disabled")
		return nil, nil
	}

	// check to see if TLS is enabled with a simple truthy value
	if ok := getBool(config, types.ConfigTLS, roots...); ok {
		f(types.ConfigTLS, true)
		newTLS(types.ConfigTLS, true)

	} else if v := getString(config, types.ConfigTLS, roots...); v != "" {
		// check to see if TLS is disabled
		if strings.EqualFold(v, "false") {
			f(types.ConfigTLS, "false")
			ctx.WithField(types.ConfigTLS, "false").Info("tls disabled")
			return nil, nil
		}

		// check to see if TLS is enabled with insecure
		if strings.EqualFold(v, "insecure") {
			f(types.ConfigTLS, "insecure")
			newTLS(types.ConfigTLS, "insecure")
			tlsConfig.InsecureSkipVerify = true

			// check to see if TLS is enabled with peers
		} else if strings.EqualFold(v, "verifyPeers") {
			f(types.ConfigTLS, "verifyPeers")
			newTLS(types.ConfigTLS, "verifyPeers")
			tlsConfig.InsecureSkipVerify = true
			tlsConfig.VerifyPeers = true

			// check to see if TLS is enabled with an expected sha256
			// fingerprint
		} else if kh, err := ParseKnownHost(ctx, v); err != nil {
			ctx.Error(err)
			return nil, err
		} else if kh != nil {
			f(types.ConfigTLS, kh.String())
			newTLS(types.ConfigTLS, kh.String())
			tlsConfig.InsecureSkipVerify = true
			tlsConfig.VerifyPeers = true
			tlsConfig.KnownHost = kh
		}
	}

	// always check for the user's known_hosts file
	if gotil.FileExists(pathConfig.UserDefaultTLSKnownHosts) {
		newTLS("usrKnownHosts", pathConfig.UserDefaultTLSKnownHosts)
		f("usrKnownHosts", pathConfig.UserDefaultTLSKnownHosts)
		tlsConfig.UsrKnownHosts = pathConfig.UserDefaultTLSKnownHosts
		tlsConfig.InsecureSkipVerify = true
		tlsConfig.VerifyPeers = true
	}

	if getBool(config, types.ConfigTLSInsecure, roots...) {
		newTLS(types.ConfigTLSInsecure, true)
		f(types.ConfigTLSInsecure, true)
		tlsConfig.InsecureSkipVerify = true
	}

{'fraction_non_alphanumeric': 0.1404539385847797, 'fraction_numerical': 0.003738317757009346, 'mean_word_length': 3.9089121887287024, 'pattern_counts': {'":': 8, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4973331', 'n_tokens_mistral': 2920, 'n_tokens_neox': 2654, 'n_words': 633}