<commit_before>package main

import (
	"bytes"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/dustin/go-humanize"
	"github.com/kothar/brotli-go/dec"
	"github.com/kothar/brotli-go/enc"
)

func testBrotli() {
	start := time.Now()

	src := os.Args[2]
	inputBuffer, err := ioutil.ReadFile(src)
	if err != nil {
		panic(err)
	}

	log.Println("Read file in", time.Since(start))
	log.Println("Uncompressed size is", humanize.Bytes(uint64(len(inputBuffer))))
	start = time.Now()

	var decoded []byte

	for q := 0; q <= 9; q++ {
		params := enc.NewBrotliParams()
		params.SetQuality(q)

		encoded, err := enc.CompressBuffer(params, inputBuffer, make([]byte, 1))
		if err != nil {
			panic(err)
		}

		log.Println("Compressed (q=", q, ") to", humanize.Bytes(uint64(len(encoded))), "in", time.Since(start))
		start = time.Now()

		decoded, err = dec.DecompressBuffer(encoded, make([]byte, 1))
		if err != nil {
			panic(err)
		}

		log.Println("Decompressed in", time.Since(start))
		start = time.Now()
	}

	if !bytes.Equal(inputBuffer, decoded) {
		log.Println("Decoded output does not match original input")
		return
	}

	log.Println("Compared in", time.Since(start))
	start = time.Now()

	log.Println("Round-trip through brotli successful!")
}
<commit_msg>Use new brotli-go streaming API<commit_after>package main

import (
	"bytes"
	"crypto/md5"
	"io"
	"log"
	"os"
	"time"

	"github.com/dustin/go-humanize"
	"gopkg.in/kothar/brotli-go.v0/dec"
	"gopkg.in/kothar/brotli-go.v0/enc"
)

type counterWriter struct {
	count  int64
	writer io.Writer
}

func (w *counterWriter) Write(buffer []byte) (int, error) {
	written, err := w.writer.Write(buffer)
	w.count += int64(written)
	return written, err
}

func (w *counterWriter) Close() error {
	if v, ok := w.writer.(io.Closer); ok {
		return v.Close()
	}

	return nil
}

func testBrotli() {
	start := time.Now()

	src := os.Args[2]

	stats, err := os.Lstat(src)
	if err != nil {
		panic(err)
	}

	log.Println("Uncompressed size is", stats.Size(), "bytes =", humanize.Bytes(uint64(stats.Size())))

	srcReader, err := os.Open(src)
	if err != nil {
		panic(err)
	}

	h := md5.New()
	io.Copy(h, srcReader)

	originalHash := h.Sum(nil)
	log.Printf("Uncompressed hash is %x\n", originalHash)

	start = time.Now()

	params := enc.NewBrotliParams()

	for q := 0; q <= 9; q++ {
		pr, pw := io.Pipe()
		cw := &counterWriter{writer: pw}

		var compressedSize int64

		go func() {
			params.SetQuality(q)
			writer := enc.NewBrotliWriter(params, cw)
			defer writer.Close()

			srcReader.Seek(0, os.SEEK_SET)
			_, err = io.Copy(writer, srcReader)
			if err != nil {
				panic(err)
			}
		}()

		reader := dec.NewBrotliReader(pr)
		h.Reset()

		_, err = io.Copy(h, reader)
		if err != nil {
			panic(err)
		}

		decodedHash := h.Sum(nil)

		compressedSize = cw.count
		log.Println("{,de}compressed (q=", q, ") to", humanize.Bytes(uint64(compressedSize)), "in", time.Since(start))

		if !bytes.Equal(originalHash, decodedHash) {
			log.Printf("Decoded output does not match original input: %x\n", decodedHash)
			return
		}
		start = time.Now()
	}

	log.Println("Round-trip through brotli successful!")
}

{'fraction_non_alphanumeric': 0.1725790987535954, 'fraction_numerical': 0.00862895493767977, 'mean_word_length': 3.735249621785174, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4835181', 'n_tokens_mistral': 1358, 'n_tokens_neox': 1242, 'n_words': 264}