<commit_before>package main

import (
	"io"
	"os"

	lz4 "github.com/bkaradzic/go-lz4"
)

type Lz4Writer struct {
	Writer        io.Writer
	MaxBufferSize uint32
	HeaderWritten bool
}

// GenerateLz4Header creates a valid LZ4 1.5 Header.
// https://docs.google.com/document/d/
// 1cl8N1bmkTdIpPLtnlzbBSFAdUeyNo5fwfHbHU7VRNWY/edit?pli=1
func GenerateLz4Header() []byte {

	header := []byte{
		// LZ4 Magic number, Little Endian (4 Bytes):
		0x04, 0x22, 0x4d, 0x18,
		// Frame Descriptor (3-11 Bytes):
		0x0, 0x1, 0x3,
	}
	return header
}

func (w *Lz4Writer) writeHeader() {
	if _, err := w.Writer.Write(GenerateLz4Header()); err != nil {
		panic(err)
	}
	w.HeaderWritten = true
}

func (w *Lz4Writer) Write(data []byte) (int, error) {
	dst := make([]byte, len(data))
	compressedBytes, err := lz4.Encode(dst, data)
	if err != nil {
		return 0, err
	}

	if !w.HeaderWritten {
		w.writeHeader()
	}

	bytes, err := w.Writer.Write(compressedBytes)
	return bytes, err
}

func main() {
	fdFrom, _ := os.OpenFile(os.Args[1], os.O_RDONLY, 0644)
	defer fdFrom.Close()
	fdTo, _ := os.OpenFile(os.Args[2], os.O_CREATE|os.O_WRONLY, 0644)
	defer fdTo.Close()
	writer := &Lz4Writer{
		Writer:        fdTo,
		MaxBufferSize: lz4.MaxInputSize,
	}

	buffer := make([]byte, 4096)
	for {
		read, err := fdFrom.Read(buffer)
		if err != nil {
			break
		}
		writer.Write(buffer[:read])
	}
}<commit_msg>compression.go: Migrated from LZ4 to Snappy.<commit_after>package main

import (
	"github.com/golang/snappy"
	"io"
	"os"
)

// Compress the file at src to dst.
func CompressFile(src, dst string) (int64, error) {
	fdFrom, err := os.OpenFile(src, os.O_RDONLY, 0644)
	if err != nil {
		return 0, err
	}
	defer fdFrom.Close()

	fdTo, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return 0, err
	}
	defer fdTo.Close()

	return Compress(fdFrom, fdTo)
}

// Decompress the file at src to dst.
func DecompressFile(src, dst string) (int64, error) {
	fdFrom, err := os.OpenFile(src, os.O_RDONLY, 0644)
	if err != nil {
		return 0, err
	}
	defer fdFrom.Close()

	fdTo, err := os.OpenFile(dst, os.O_CREATE|os.O_WRONLY, 0644)
	if err != nil {
		return 0, err
	}
	defer fdTo.Close()

	return Decompress(fdFrom, fdTo)
}

// Compress represents a layer stream compression.
// As input src and dst io.Reader/io.Writer is expected.
func Compress(src io.Reader, dst io.Writer) (int64, error) {
	return io.Copy(snappy.NewWriter(dst), src)
}

// Decompress represents a layer for stream decompression.
// As input src and dst io.Reader/io.Writer is expected.
func Decompress(src io.Reader, dst io.Writer) (int64, error) {
	return io.Copy(dst, snappy.NewReader(src))
}

// NewReader returns a new compression Reader.
func NewReader(r io.Reader) io.Reader {
	return snappy.NewReader(r)
}

// NewWriter returns a new compression Writer.
func NewWriter(w io.Writer) io.Writer {
	return snappy.NewWriter(w)
}

func main() {

	if os.Args[1] == "d" {
		DecompressFile(os.Args[2], os.Args[3])
	}

	if os.Args[1] == "c" {
		CompressFile(os.Args[2], os.Args[3])
	}
}
{'fraction_non_alphanumeric': 0.14638157894736842, 'fraction_numerical': 0.030921052631578946, 'mean_word_length': 3.7001545595054095, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3929150', 'n_tokens_mistral': 1304, 'n_tokens_neox': 1218, 'n_words': 303}