<commit_before>package main

import (
	"bytes"
	"fmt"
	"os"
	"log"
	"path/filepath"
	"os/exec"
	"github.com/mgutz/ansi"
	"strings"
)

type SiteList struct {
	Path  string
	Sites []string
}

func inList(item string, list []string) bool {
	for _, i := range list {
		if i == item {
			return true
		}
	}
	return false
}

func main() {

	var e error

	if len(os.Args) < 2 {
		log.Fatal("Not enough arguments.")
	}

	var S SiteList
	
	blacklist := []string{"all", "default"}

	S.Path, e = os.Getwd()
	if e != nil {
		log.Fatal(e)
	}

	module := os.Args[1]

	// var visit = func(p string, f os.FileInfo, e error) error {
	// 	if f.IsDir() && p != S.Path {
	// 		_, file := filepath.Split(p)
	// 		if !inList(file, blacklist) {
	// 			S.Sites = append(S.Sites, file)
	// 		}
	// 	}
	// 	return nil
	// }
	// if e := filepath.Walk(S.Path, visit); e != nil {
	// 	log.Fatal(e)
	// }

	// Get all file paths in the current working directory.
	files, e := filepath.Glob(fmt.Sprintf("%s/*", S.Path))
	if e != nil {
		log.Fatal(e)
	}

	// Loop over files and append only directories, that are not blacklisted, to S.Sites.
	for _, file := range files {
		_, f := filepath.Split(file)

		finfo, e := os.Lstat(file)
		if e != nil {
			log.Fatal(e)
		}

		if finfo.IsDir() && !inList(f, blacklist) {
			S.Sites = append(S.Sites, f)
		}
	}

	var tcount, encount, dcount int

	green := ansi.ColorCode("green+h:black")
	red := ansi.ColorCode("red+h:black")<commit_msg>Removed some old code.<commit_after>package main

import (
	"bytes"
	"fmt"
	"os"
	"log"
	"path/filepath"
	"os/exec"
	"github.com/mgutz/ansi"
	"strings"
)

type SiteList struct {
	Path  string
	Sites []string
}

func inList(item string, list []string) bool {
	for _, i := range list {
		if i == item {
			return true
		}
	}
	return false
}

func main() {

	var e error

	if len(os.Args) < 2 {
		log.Fatal("Not enough arguments.")
	}

	var S SiteList
	
	blacklist := []string{"all", "default"}

	S.Path, e = os.Getwd()
	if e != nil {
		log.Fatal(e)
	}

	module := os.Args[1]

	// Get all file paths in the current working directory.
	files, e := filepath.Glob(fmt.Sprintf("%s/*", S.Path))
	if e != nil {
		log.Fatal(e)
	}

	// Loop over files and append only directories, that are not blacklisted, to S.Sites.
	for _, file := range files {
		_, f := filepath.Split(file)

		finfo, e := os.Lstat(file)
		if e != nil {
			log.Fatal(e)
		}

		if finfo.IsDir() && !inList(f, blacklist) {
			S.Sites = append(S.Sites, f)
		}
	}

	var tcount, encount, dcount int

	green := ansi.ColorCode("green+h:black")
	red := ansi.ColorCode("red+h:black")
{'fraction_non_alphanumeric': 0.17105263157894737, 'fraction_numerical': 0.0015479876160990713, 'mean_word_length': 2.82396449704142, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '721652', 'n_tokens_mistral': 1131, 'n_tokens_neox': 1050, 'n_words': 272}