<commit_before>package main

import (
	"fmt"
	"os"
	"path/filepath"
)

var encoder map[string]string = map[string]string{
	"a": "u",
	"b": "q",
	"c": "z",
	"d": "n",
	"e": "o",
	"f": "b",
	"g": "s",
	"h": "x",
	"i": "a",
	"j": "d",
	"k": "y",
	"l": "p",
	"m": "r",
	"n": "h",
	"o": "e",
	"p": "w",
	"q": "v",
	"r": "c",
	"s": "t",
	"t": "l",
	"u": "i",
	"v": "f",
	"w": "g",
	"x": "j",
	"y": "m",
	"z": "k",
}

var decoder map[string]string = map[string]string{
	"u": "a",
	"q": "b",
	"z": "c",
	"n": "d",
	"o": "e",
	"b": "f",
	"s": "g",
	"x": "h",
	"a": "i",
	"d": "j",
	"y": "k",
	"p": "l",
	"r": "m",
	"h": "n",
	"e": "o",
	"w": "p",
	"v": "q",
	"c": "r",
	"t": "s",
	"l": "t",
	"i": "u",
	"f": "v",
	"g": "w",
	"j": "x",
	"m": "y",
	"k": "z",
}

func main() {
	args := os.Args

	if len(args) < 2 {
		fail()
	}

	if args[1] != "e" && args[1] != "d" {
		fail()
	}

	if args[1] == "e" {
		filepath.Walk(".", walkEncode)
	} else {
		filepath.Walk(".", walkDecode)
	}
}

func fail() {
	fmt.Println("Usage: tis e|d")
	os.Exit(1)
}

func isHidden(filepath string) bool {
	if len(filepath) > 1 && string(filepath[0]) == "." {
		return true
	}

	return false
}

func walkDecode(path string, info os.FileInfo, err error) error {
	if isHidden(path) {
		return filepath.SkipDir
	}

	if info.IsDir() {
		fmt.Printf("%s => %s\n", path, codec(path, decoder))
	}
	return nil
}

func walkEncode(path string, info os.FileInfo, err error) error {
	if isHidden(path) {
		return filepath.SkipDir
	}

	if info.IsDir() {
		fmt.Printf("%s => %s\n", path, codec(path, encoder))
	}
	return nil
}

func codec(in string, mapper map[string]string) string {

	ret := ""

	for _, letter := range in {
		l := string(letter)
		r, ok := mapper[l]

		if ok {
			ret += r
		} else {
			ret += l
		}
	}

	return ret
}<commit_msg>adding logic for file extensions<commit_after>package main

import (
	"fmt"
	"os"
	"path/filepath"
	"strings"
)

type cipher map[string]string

var encoder cipher = map[string]string{
	"a": "u",
	"b": "q",
	"c": "z",
	"d": "n",
	"e": "o",
	"f": "b",
	"g": "s",
	"h": "x",
	"i": "a",
	"j": "d",
	"k": "y",
	"l": "p",
	"m": "r",
	"n": "h",
	"o": "e",
	"p": "w",
	"q": "v",
	"r": "c",
	"s": "t",
	"t": "l",
	"u": "i",
	"v": "f",
	"w": "g",
	"x": "j",
	"y": "m",
	"z": "k",
}

var decoder cipher = map[string]string{
	"u": "a",
	"q": "b",
	"z": "c",
	"n": "d",
	"o": "e",
	"b": "f",
	"s": "g",
	"x": "h",
	"a": "i",
	"d": "j",
	"y": "k",
	"p": "l",
	"r": "m",
	"h": "n",
	"e": "o",
	"w": "p",
	"v": "q",
	"c": "r",
	"t": "s",
	"l": "t",
	"i": "u",
	"f": "v",
	"g": "w",
	"j": "x",
	"m": "y",
	"k": "z",
}

var debug bool

func main() {
	args := os.Args

	if len(args) < 2 {
		fail()
	}

	if args[1] != "e" && args[1] != "d" {
		fail()
	}

	debug = checkDebug(args)

	if args[1] == "e" {
		err := filepath.Walk(".", walkEncode)

		if err != nil {
			fmt.Println("encode err: ", err.Error())
		}
	} else {
		err := filepath.Walk(".", walkDecode)

		if err != nil {
			fmt.Println("decode err: ", err.Error())
		}
	}
}

func fail() {
	fmt.Println("Usage: tis e|d")
	os.Exit(1)
}

func isHidden(filename string) bool {
	if len(filename) > 1 && string(filename[0]) == "." {
		return true
	}

	return false
}

func walkDecode(path string, info os.FileInfo, err error) error {
	if err != nil {
		fmt.Printf("Error on path: %s, %s\n", path, err.Error())
		return err
	}

	return printPath(path, info, decoder)
}

func walkEncode(path string, info os.FileInfo, err error) error {
	if err != nil {
		fmt.Printf("Error on path: %s, %s\n", path, err.Error())
		return err
	}

	return printPath(path, info, encoder)
}

func printPath(path string, info os.FileInfo, c cipher) error {
	// check if this is a hidden file/directory, if so we want to skip it
	if isHidden(info.Name()) {
		if debug {
			fmt.Printf("Skipping hidden path: %s\n", path)
		}
		if info.IsDir() {
			return filepath.SkipDir
		}

		return nil
	} else {
		// translate the encoded/decoded representation of this file/dir
		// but, do not translate the file's extension
		p, ext := splitExtension(path)

		str := translate(p, c)
		str += ext
		fmt.Printf("%s => %s\n", path, str)
	}

	return nil
}

func splitExtension(path string) (string, string) {
	ext := filepath.Ext(path)
	if ext == "" {
		return path, ""
	}

	parts := strings.Split(path, ext)
	if len(parts) > 2 {
		// combine into one path before extension
		path = strings.Join(parts[0:len(parts)-1], "")
	} else {
		path = parts[0]
	}

	return path, ext
}

func translate(in string, mapper cipher) string {
	ret := ""

	for _, letter := range in {
		l := string(letter)
		r, ok := mapper[l]

		if ok {
			ret += r
		} else {
			ret += l
		}
	}

	return ret
}

func checkDebug(args []string) bool {
	for _, arg := range args {
		if arg == "--debug" {
			return true
		}
	}

	return false
}
{'fraction_non_alphanumeric': 0.2542726135889954, 'fraction_numerical': 0.003751563151313047, 'mean_word_length': 3.0395622895622894, 'pattern_counts': {'":': 104, '<': 5, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4166923', 'n_tokens_mistral': 2336, 'n_tokens_neox': 2222, 'n_words': 595}