<commit_before>package compressor

import (
	"archive/tar"
	"compress/gzip"
	"io"
	"os"
	"path/filepath"
)

type Compressor interface {
	Compress(src string, dest string) error
}

func NewTgz() Compressor {
	return &tgzCompressor{}
}

type tgzCompressor struct{}

func (compressor *tgzCompressor) Compress(src string, dest string) error {
	absPath, err := filepath.Abs(src)
	if err != nil {
		return err
	}

	file, err := os.Open(absPath)
	if err != nil {
		return err
	}
	defer file.Close()

	// file write
	fw, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer fw.Close()

	gw := gzip.NewWriter(fw)
	defer gw.Close()

	tw := tar.NewWriter(gw)
	defer tw.Close()

	return compressor.compressRecursively(file, absPath, tw)
}

func (compressor *tgzCompressor) compressRecursively(file *os.File, relativeFrom string, tw *tar.Writer) error {
	info, err := os.Lstat(file.Name())
	if err != nil {
		return err
	}

	if info.IsDir() {
		files, err := file.Readdir(0)
		if err != nil {
			return err
		}

		// Add subfolders as entries.  This even works for empty dirs.
		if relativeFrom != file.Name() {
			hdr, err := tar.FileInfoHeader(info, "")
			if err != nil {
				return err
			}

			err = tw.WriteHeader(hdr)
			if err != nil {
				return err
			}
		}

		for _, info := range files {
			subName := filepath.Join(file.Name(), info.Name())

			subFile, err := os.Open(subName)
			if err != nil {
				return err
			}

			err = compressor.compressRecursively(subFile, relativeFrom, tw)
			if err != nil {
				return err
			}
		}
	} else {
		err = compressor.addFileToTar(file, info, relativeFrom, tw)
		if err != nil {
			return err
		}
	}

	return nil
}

func (compressor *tgzCompressor) addFileToTar(file *os.File, info os.FileInfo, relativeFrom string, tw *tar.Writer) error {
	link, err := os.Readlink(file.Name())
	if err != nil {
		link = ""
	}

	h, err := tar.FileInfoHeader(info, link)
	if err != nil {
		return err
	}

	relative, err := filepath.Rel(relativeFrom, file.Name())
	if err != nil {
		return err
	}

	if relative == "." {
		relative = info.Name()
	}

	h.Name = relative

	err = tw.WriteHeader(h)
	if err != nil {
		return err
	}

	if link == "" {
		_, err := io.Copy(tw, file)
		if err != nil {
			return err
		}
	}

	return nil
}<commit_msg>use filepath.Walk, refactor a bit<commit_after>package compressor

import (
	"archive/tar"
	"compress/gzip"
	"io"
	"os"
	"path/filepath"
	"strings"
)

type Compressor interface {
	Compress(src string, dst string) error
}

func NewTgz() Compressor {
	return &tgzCompressor{}
}

type tgzCompressor struct{}

func (compressor *tgzCompressor) Compress(src string, dest string) error {
	absPath, err := filepath.Abs(src)
	if err != nil {
		return err
	}

	file, err := os.Open(absPath)
	if err != nil {
		return err
	}
	defer file.Close()

	fw, err := os.Create(dest)
	if err != nil {
		return err
	}
	defer fw.Close()

	gw := gzip.NewWriter(fw)
	defer gw.Close()

	tw := tar.NewWriter(gw)
	defer tw.Close()

	return filepath.Walk(absPath, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		relative, err := filepath.Rel(absPath, path)
		if err != nil {
			return err
		}

		return addTarFile(path, relative, tw)
	})
}

func addTarFile(path, name string, tw *tar.Writer) error {
	fi, err := os.Lstat(path)
	if err != nil {
		return err
	}

	link := ""
	if fi.Mode()&os.ModeSymlink != 0 {
		if link, err = os.Readlink(path); err != nil {
			return err
		}
	}

	hdr, err := tar.FileInfoHeader(fi, link)
	if err != nil {
		return err
	}

	if fi.IsDir() && !strings.HasSuffix(name, "/") {
		name = name + "/"
	}

	if hdr.Typeflag == tar.TypeReg && name == "." {
		// archiving a single file
		hdr.Name = filepath.Base(path)
	} else {
		hdr.Name = name
	}

	if err := tw.WriteHeader(hdr); err != nil {
		return err
	}

	if hdr.Typeflag == tar.TypeReg {
		file, err := os.Open(path)
		if err != nil {
			return err
		}

		defer file.Close()

		_, err = io.Copy(tw, file)
		if err != nil {
			return err
		}
	}

	return nil
}
{'fraction_non_alphanumeric': 0.14328284082443507, 'fraction_numerical': 0.0004966476285075739, 'mean_word_length': 3.0199600798403194, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6787128', 'n_tokens_mistral': 1678, 'n_tokens_neox': 1512, 'n_words': 413}