<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/ftrvxmtrx/tga"
	"image/png"
	"io/ioutil"
	"os"
	"path/filepath"
	// "runtime"
	"strings"
	"sync"
	"sync/atomic"
)

var wg sync.WaitGroup
var filecount int64
var dircount int64
var errorcount int64

//path values from flags
var path string
var outdir string

func main() {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered from panic: ", r)
		}
	}()
	doflags()

	walkdir("/Users/Josh")
	wg.Wait()
	fmt.Printf("Found %d file %d directories and encountered %d errors", filecount, dircount, errorcount)
}

func walkdir(dir string) {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		fmt.Println("Error at: ", dir, "\n>>", err.Error())
		atomic.AddInt64(&errorcount, 1)
		return
	}

	for _, v := range files {
		if v.IsDir() {
			newdir := filepath.Join(dir, v.Name())
			atomic.AddInt64(&dircount, 1)
			walkdir(newdir)
		} else {
			wg.Add(1)
			atomic.AddInt64(&filecount, 1)
			go func(f os.FileInfo, dir string) {
				defer wg.Done()
				name := filepath.Join(dir, f.Name())
				fmt.Println(name)
				//call a convert method here passing in the filename "name"
			}(v, dir)
		}
	}
}

func doflags() {
	flag.StringVar(&path, "path", "", "Folder with targa files")
	flag.StringVar(&outdir, "out", "", "Output folder where png files will be saved")
	inplace := flag.Bool("i", false, "If set, will ignore output directory and convert files inplace")
	flag.Parse()

	//check flags
	if f, err := os.Stat(path); os.IsNotExist(err) {
		panic(err)
	} else if err != nil {
		panic(err)
	} else if !f.IsDir() {
		panic(f.Name() + " is not a valid directory!")
	}

}

//converttga should always be called in a new go function
func converttga(fname string) {
	defer wg.Done()
	file, err := os.Open(fname)
	if err != nil {
		fmt.Println("Could not open file: ", fname, "\n>>", err.Error())
		return
	}
	img, err := tga.Decode(file)
	if err != nil {
		fmt.Println("Error decoding file: ", fname, "\n>>", err.Error())
		return
	}
}

func getfinaldir(fname string) string {
	return "" + "hi"
}<commit_msg>Finshed helper function getfinaldir. Finshed function converttga.<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/ftrvxmtrx/tga"
	"image/png"
	"io/ioutil"
	"os"
	"path/filepath"
	// "runtime"
	// "strings"
	"sync"
	"sync/atomic"
)

var wg sync.WaitGroup
var filecount int64
var dircount int64
var errorcount int64

//path values from flags
var path string
var outdir string

func main() {
	defer func() {
		if r := recover(); r != nil {
			fmt.Println("Recovered from panic: ", r)
		}
	}()
	doflags()

	walkdir("/Users/Josh")
	wg.Wait()
	fmt.Printf("Found %d file %d directories and encountered %d errors", filecount, dircount, errorcount)
}

func walkdir(dir string) {
	files, err := ioutil.ReadDir(dir)
	if err != nil {
		fmt.Println("Error at: ", dir, "\n>>", err.Error())
		atomic.AddInt64(&errorcount, 1)
		return
	}

	for _, v := range files {
		if v.IsDir() {
			newdir := filepath.Join(dir, v.Name())
			atomic.AddInt64(&dircount, 1)
			walkdir(newdir)
		} else {
			wg.Add(1)
			atomic.AddInt64(&filecount, 1)
			go func(f os.FileInfo, dir string) {
				defer wg.Done()
				name := filepath.Join(dir, f.Name())
				fmt.Println(name)
				//call a convert method here passing in the filename "name"
				wg.Add(1)
				go converttga(name)
			}(v, dir)
		}
	}
}

func doflags() {
	flag.StringVar(&path, "path", "", "Folder with targa files")
	flag.StringVar(&outdir, "out", "", "Output folder where png files will be saved")
	flag.Parse()

	//check flags
	if f, err := os.Stat(path); os.IsNotExist(err) {
		panic(err)
	} else if err != nil {
		panic(err)
	} else if !f.IsDir() {
		panic(f.Name() + " is not a valid directory!")
	}

	if outdir == "" {
		outdir = path
	} else if _, err := os.Stat(outdir); err != nil {
		panic(err)
	}
}

//converttga should always be called in a new go function
func converttga(fname string) {
	defer wg.Done()
	file, err := os.Open(fname)
	if err != nil {
		fmt.Println("Could not open file: ", fname, "\n>>", err.Error())
		return
	}

	img, err := tga.Decode(file)
	if err != nil {
		fmt.Println("Error decoding file: ", fname, "\n>>", err.Error())
		return
	}

	newname := getfinaldir(fname)
	pngfile, err := os.Create(newname)
	if err != nil {
		fmt.Println("Error, could not create file: ", newname)
		return
	}
	err = png.Encode(pngfile, img)
	if err != nil {
		fmt.Println("Could not Encode file to png: ", fname)
	}
}

func getfinaldir(fname string) string {
	partialp := fname[len(path):len(fname)-len(filepath.Ext(fname))] + ".png"
	newpath := filepath.Join(outdir, partialp)
	// fmt.Println(newpath)
	return newpath
}
{'fraction_non_alphanumeric': 0.16341411108728288, 'fraction_numerical': 0.007076989062835085, 'mean_word_length': 3.5193798449612403, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 15, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1002774', 'n_tokens_mistral': 1930, 'n_tokens_neox': 1795, 'n_words': 486}