<commit_before>package discover

import (
	"os"
	"fmt"
	"bufio"
	"regexp"
	"strings"
	"path/filepath"
)

var (
	root string
	tree []string
	languages = []language{
		puppetFiles,
		puppetModule,
		ansibleRoles,
		ansiblePlaybooks,
	}
)

type language struct {
	Key		string
	Extensions	[]string
	Paths		[]string
	PathMatchers	[]string
	ContentRegex	[]string
	IgnoredDirs	[]string
}

// Run returns all matches of a language type from the root of the specified tree
func Run(r string) map[string][]string {
	if ! strings.HasSuffix(r, "/") {
		r = fmt.Sprintf("%s/", r)
	}
	root = r
	os.Chdir(root)
	walkDirectory(root)
	results := analyzeTree(languages, tree)
	results["root"] = []string{root}
	return results
}

func walkDirectory(root string) {
	filepath.Walk(root, visitFile)
}

func visitFile(path string, file os.FileInfo, err error) error {
	if !file.IsDir() {
		rpath := path
		if root != "." {
			rpath = strings.Replace(path, root, "", 1)
		}
		tree = append(tree, rpath)
	}
	return nil
}

func analyzeTree(languages []language, tree []string) map[string][]string {
	var matches []string
	var candidates = make(map[string][]string)
	for _, f := range tree {
		for _, lang := range languages {
			match := matchLanguage(lang, f)
			if match != "" {
				matches = append(matches, match)
				candidates[lang.Key] = append(candidates[lang.Key], match)
			}
		}
	}
	for _, lang := range languages {
		candidates[lang.Key] = deduplicate(candidates[lang.Key], matches)
	}
	return candidates
}

func deduplicate(languageMatches []string, totalMatches []string) []string {
	var results []string
	var found bool
	for _, l := range languageMatches {
		found = false
		for _, m := range totalMatches {
			if strings.Contains(l, m) && l != m {
				found = true
			}
		}
		if !found && !stringInSlice(l, results) {
			results = append(results, l)
		}
	}
	return results
}

func matchLanguage(lang language, file string) string {
	for _, value := range lang.IgnoredDirs {
		if search(value, file) {
			return ""
		}
	}
	for _, value := range lang.Paths {
		if search(value, file) {
			return strings.Replace(file, value, "", 1)
		}
	}
	for _, value := range lang.PathMatchers {
		match, _ := regexp.MatchString(fmt.Sprintf("^.*(/)?%s$", value), file)
		if match {
			re := regexp.MustCompile("(" + value + ")")
			return re.ReplaceAllString(file, "")
		}
	}
	for _, value := range lang.Extensions {
		ext := strings.Replace(filepath.Ext(file), ".", "", 1)
		if value == ext {
			return file
		}

	}
	for _, value := range lang.ContentRegex {
		if searchContent(value, file) {
			return strings.Replace(file, value, "", 1)
		}
	}
	return ""
}

func search(value string, target string) bool {
	exp := fmt.Sprintf("^.*(/)?%s(/)?", value)
	matched, _ := regexp.MatchString(exp, target)
	if matched {
		return true
	}
	return false
}

func stringInSlice(value string, slice []string) bool {
	for _, s := range slice {
		if s == value {
			return true
		}
	}
	return false
}

func searchContent(expression string, file string) bool {
	regex, err := regexp.Compile(expression)
	if err != nil {
		return false
	}
	fh, err := os.Open(file)
	f := bufio.NewReader(fh)
	if err != nil {
		return false
	}
	defer fh.Close()
	buf := make([]byte, 1024)
	for {
		buf, _ , err = f.ReadLine()
		if err != nil {
			return false
		}
		s := string(buf)
		if regex.MatchString(s) {
			return true
		}
	}
	return false
}<commit_msg>Change back to CWD after walking root path<commit_after>package discover

import (
	"os"
	"fmt"
	"bufio"
	"regexp"
	"strings"
	"path/filepath"
)

var (
	root string
	tree []string
	results map[string][]string
	languages = []language{
		puppetFiles,
		puppetModule,
		ansibleRoles,
		ansiblePlaybooks,
	}
)

type language struct {
	Key		string
	Extensions	[]string
	Paths		[]string
	PathMatchers	[]string
	ContentRegex	[]string
	IgnoredDirs	[]string
}

// Run returns all matches of a language type from the root of the specified tree
func Run(r string) map[string][]string {
	if ! strings.HasSuffix(r, "/") {
		r = fmt.Sprintf("%s/", r)
	}
	root = r
	wd, err := os.Getwd()
	if err != nil {
		results["error"] = []string{err.Error()}
	} else {
		os.Chdir(root)
		walkDirectory(root)
		results = analyzeTree(languages, tree)
		os.Chdir(wd)
	}
	results["root"] = []string{root}
	return results
}

func walkDirectory(root string) {
	filepath.Walk(root, visitFile)
}

func visitFile(path string, file os.FileInfo, err error) error {
	if !file.IsDir() {
		rpath := path
		if root != "." {
			rpath = strings.Replace(path, root, "", 1)
		}
		tree = append(tree, rpath)
	}
	return nil
}

func analyzeTree(languages []language, tree []string) map[string][]string {
	var matches []string
	var candidates = make(map[string][]string)
	for _, f := range tree {
		for _, lang := range languages {
			match := matchLanguage(lang, f)
			if match != "" {
				matches = append(matches, match)
				candidates[lang.Key] = append(candidates[lang.Key], match)
			}
		}
	}
	for _, lang := range languages {
		candidates[lang.Key] = deduplicate(candidates[lang.Key], matches)
	}
	return candidates
}

func deduplicate(languageMatches []string, totalMatches []string) []string {
	var results []string
	var found bool
	for _, l := range languageMatches {
		found = false
		for _, m := range totalMatches {
			if strings.Contains(l, m) && l != m {
				found = true
			}
		}
		if !found && !stringInSlice(l, results) {
			results = append(results, l)
		}
	}
	return results
}

func matchLanguage(lang language, file string) string {
	for _, value := range lang.IgnoredDirs {
		if search(value, file) {
			return ""
		}
	}
	for _, value := range lang.Paths {
		if search(value, file) {
			return strings.Replace(file, value, "", 1)
		}
	}
	for _, value := range lang.PathMatchers {
		match, _ := regexp.MatchString(fmt.Sprintf("^.*(/)?%s$", value), file)
		if match {
			re := regexp.MustCompile("(" + value + ")")
			return re.ReplaceAllString(file, "")
		}
	}
	for _, value := range lang.Extensions {
		ext := strings.Replace(filepath.Ext(file), ".", "", 1)
		if value == ext {
			return file
		}

	}
	for _, value := range lang.ContentRegex {
		if searchContent(value, file) {
			return strings.Replace(file, value, "", 1)
		}
	}
	return ""
}

func search(value string, target string) bool {
	exp := fmt.Sprintf("^.*(/)?%s(/)?", value)
	matched, _ := regexp.MatchString(exp, target)
	if matched {
		return true
	}
	return false
}

func stringInSlice(value string, slice []string) bool {
	for _, s := range slice {
		if s == value {
			return true
		}
	}
	return false
}

func searchContent(expression string, file string) bool {
	regex, err := regexp.Compile(expression)
	if err != nil {
		return false
	}
	fh, err := os.Open(file)
	f := bufio.NewReader(fh)
	if err != nil {
		return false
	}
	defer fh.Close()
	buf := make([]byte, 1024)
	for {
		buf, _ , err = f.ReadLine()
		if err != nil {
			return false
		}
		s := string(buf)
		if regex.MatchString(s) {
			return true
		}
		return false
	}
}
{'fraction_non_alphanumeric': 0.1474469305794607, 'fraction_numerical': 0.002294893861158921, 'mean_word_length': 3.50161394448031, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6217996', 'n_tokens_mistral': 2677, 'n_tokens_neox': 2468, 'n_words': 661}