<commit_before>package packing

import (
	"archive/tar"
	"compress/gzip"
	"github.com/tywkeene/autobd/options"
	"io"
	"log"
	"os"
	"path/filepath"
)

func UnpackDir(source io.Reader) error {
	//Unzip the contents first
	gr, err := gzip.NewReader(source)
	if err != nil {
		return err
	}
	defer gr.Close()
	tr := tar.NewReader(gr)

	for {
		header, err := tr.Next()
		if err != nil {
			return err
		} else if err == io.EOF {
			break
		}

		filename := header.Name
		log.Println("Unpacking: ", filename)

		switch header.Typeflag {
		case tar.TypeDir:
			err := os.MkdirAll(filename, os.FileMode(header.Mode))
			if err != nil {
				return err
			}
		case tar.TypeReg:
			writer, err := os.Create(filename)
			if err != nil {
				return err
			}

			io.Copy(writer, tr)
			if err = os.Chmod(filename, os.FileMode(header.Mode)); err != nil {
				return err
			}
			writer.Close()
		}
	}
	return nil
}

//addTarFile() and PackDir() are from https://github.com/pivotal-golang/archiver
//I was originally going to bring in the whole package as a dependency but it turns out
//the extractor package doesn't entirely work the way I thought. This works, so I'm putting it
//in here to avoid the whole dependency thing until I can fix it.
//TL;DR: This isn't my code.

func addTarFile(path string, name string, tw *tar.Writer) error {
	fi, err := os.Lstat(path)
	if err != nil {
		return err
	}

	link := ""
	if fi.Mode()&os.ModeSymlink != 0 {
		if link, err = os.Readlink(path); err != nil {
			return err
		}
	}

	hdr, err := tar.FileInfoHeader(fi, link)
	if err != nil {
		return err
	}
	if fi.IsDir() && !os.IsPathSeparator(name[len(name)-1]) {
		name = name + "/"
	}
	if hdr.Typeflag == tar.TypeReg && name == "." {
		hdr.Name = filepath.ToSlash(filepath.Base(path))
	} else {
		hdr.Name = filepath.ToSlash(path)
	}
	hdr.Name = filepath.ToSlash(name)
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	if hdr.Typeflag == tar.TypeReg {
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		_, err = io.Copy(tw, file)
		if err != nil {
			return err
		}
	}

	return nil
}

func PackDir(srcPath string, dest io.Writer) error {
	absolutePath, err := filepath.Abs(srcPath)
	if err != nil {
		return err
	}
	tw := tar.NewWriter(dest)
	defer tw.Close()

	err = filepath.Walk(absolutePath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		relativePath, err := filepath.Rel(options.Config.Root, path)
		if err != nil {
			return err
		}
		log.Println("Packing directory: ", relativePath)
		return addTarFile(path, relativePath, tw)
	})

	return err
}<commit_msg>Cleaned up log clutter in packing.go<commit_after>package packing

import (
	"archive/tar"
	"compress/gzip"
	"github.com/tywkeene/autobd/options"
	"io"
	"log"
	"os"
	"path/filepath"
)

func UnpackDir(source io.Reader) error {
	//Unzip the contents first
	gr, err := gzip.NewReader(source)
	if err != nil {
		return err
	}
	defer gr.Close()
	tr := tar.NewReader(gr)

	for {
		header, err := tr.Next()
		if err != nil {
			return err
		} else if err == io.EOF {
			break
		}

		filename := header.Name
		log.Println("Unpacking: ", filename)

		switch header.Typeflag {
		case tar.TypeDir:
			err := os.MkdirAll(filename, os.FileMode(header.Mode))
			if err != nil {
				return err
			}
		case tar.TypeReg:
			writer, err := os.Create(filename)
			if err != nil {
				return err
			}

			io.Copy(writer, tr)
			if err = os.Chmod(filename, os.FileMode(header.Mode)); err != nil {
				return err
			}
			writer.Close()
		}
	}
	return nil
}

//addTarFile() and PackDir() are from https://github.com/pivotal-golang/archiver
//I was originally going to bring in the whole package as a dependency but it turns out
//the extractor package doesn't entirely work the way I thought. This works, so I'm putting it
//in here to avoid the whole dependency thing until I can fix it.
//TL;DR: This isn't my code.

func addTarFile(path string, name string, tw *tar.Writer) error {
	fi, err := os.Lstat(path)
	if err != nil {
		return err
	}

	link := ""
	if fi.Mode()&os.ModeSymlink != 0 {
		if link, err = os.Readlink(path); err != nil {
			return err
		}
	}

	hdr, err := tar.FileInfoHeader(fi, link)
	if err != nil {
		return err
	}
	if fi.IsDir() && !os.IsPathSeparator(name[len(name)-1]) {
		name = name + "/"
	}
	if hdr.Typeflag == tar.TypeReg && name == "." {
		hdr.Name = filepath.ToSlash(filepath.Base(path))
	} else {
		hdr.Name = filepath.ToSlash(path)
	}
	hdr.Name = filepath.ToSlash(name)
	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}
	if hdr.Typeflag == tar.TypeReg {
		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		_, err = io.Copy(tw, file)
		if err != nil {
			return err
		}
	}

	return nil
}

func PackDir(srcPath string, dest io.Writer) error {
	absolutePath, err := filepath.Abs(srcPath)
	if err != nil {
		return err
	}
	tw := tar.NewWriter(dest)
	defer tw.Close()

	log.Println("Packing directory: ", srcPath)
	err = filepath.Walk(absolutePath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		relativePath, err := filepath.Rel(options.Config.Root, path)
		if err != nil {
			return err
		}
		return addTarFile(path, relativePath, tw)
	})

	return err
}
{'fraction_non_alphanumeric': 0.1394602755236837, 'fraction_numerical': 0.0007548594074353652, 'mean_word_length': 3.051987767584098, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '763480', 'n_tokens_mistral': 2143, 'n_tokens_neox': 1954, 'n_words': 551}