<commit_before>package detector

import (
	"encoding/base64"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/thoughtworks/talisman/git_repo"
	"strings"
)

var delimiters = []string{".", "-", "="}

type FileContentDetector struct {
}

func NewFileContentDetector() Detector {
	return FileContentDetector{}
}

func (contentDetector FileContentDetector) Test(additions []git_repo.Addition, ignores Ignores, result *DetectionResults) {
	for _, addition := range additions {
		if ignores.Deny(addition) {
			log.WithFields(log.Fields{
				"filePath": addition.Path,
			}).Info("Ignoring addition as it was specified to be ignored.")
			result.Ignore(addition.Path, fmt.Sprintf("%s was ignored by .talismanignore", addition.Path))
			continue
		}
		base64 := checkBase64EncodingForFile(addition.Data)
		if base64 == true {
			log.WithFields(log.Fields{
				"filePath": addition.Path,
			}).Info("Failing file as it contains a base64 encoded text.")
			result.Fail(addition.Path, fmt.Sprint("Expected file to not to contain base64 encoded texts"))
		}
	}
}

func checkBase64Encoding(s string) bool {
	if len(s) <= 4 {
		return false
	}
	_, err := base64.StdEncoding.DecodeString(s)
	return err == nil
}

func checkBase64EncodingForFile(content []byte) bool {
	s := string(content)
	for _, d := range delimiters {
		subStrings := strings.Split(s, d)
		for _, sub := range subStrings {
			if checkBase64Encoding(sub) {
				return true
			}
		}
	}
	return false
}<commit_msg>refactor file content detector a bit<commit_after>package detector

import (
	"encoding/base64"
	"fmt"
	log "github.com/Sirupsen/logrus"
	"github.com/thoughtworks/talisman/git_repo"
	"strings"
)

var delimiters = []string{".", "-", "="}

type FileContentDetector struct {
}

func NewFileContentDetector() Detector {
	return FileContentDetector{}
}

func (contentDetector FileContentDetector) Test(additions []git_repo.Addition, ignores Ignores, result *DetectionResults) {
	for _, addition := range additions {
		if ignores.Deny(addition) {
			log.WithFields(log.Fields{
				"filePath": addition.Path,
			}).Info("Ignoring addition as it was specified to be ignored.")
			result.Ignore(addition.Path, fmt.Sprintf("%s was ignored by .talismanignore", addition.Path))
			continue
		}
		base64 := checkBase64EncodingForFile(addition.Data)
		if base64 == true {
			log.WithFields(log.Fields{
				"filePath": addition.Path,
			}).Info("Failing file as it contains a base64 encoded text.")
			result.Fail(addition.Path, fmt.Sprint("Expected file to not to contain base64 encoded texts"))
		}
	}
}

func checkBase64Encoding(s string) bool {
	if len(s) <= 4 {
		return false
	}
	_, err := base64.StdEncoding.DecodeString(s)
	return err == nil
}

func checkBase64EncodingForFile(content []byte) bool {
	s := string(content)
	for _, d := range delimiters {
		subStrings := strings.Split(s, d)
		if checkEachSubString(subStrings) {
			return true
		}
	}
	return false
}

func checkEachSubString(subStrings []string) bool {
	for _, sub := range subStrings {
		if checkBase64Encoding(sub) {
			return true
		}
	}
	return false
}
{'fraction_non_alphanumeric': 0.12313674659753726, 'fraction_numerical': 0.013609850939727802, 'mean_word_length': 4.378048780487805, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5694446', 'n_tokens_mistral': 1122, 'n_tokens_neox': 999, 'n_words': 253}