<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"image"
	"image/jpeg"
	"io"
	"math/rand"
	"os"

	_ "image/gif" // register extra decoders for flexible input
	_ "image/png"
)

func recompress(in io.Reader, out io.Writer, quality int) error {
	img, _, err := image.Decode(in)
	if err != nil {
		return err
	}

	return jpeg.Encode(out, img, &jpeg.Options{Quality: quality})
}

func uglify(in io.Reader, cycles, lowerBound int) (io.Reader, error) {
	randRange := 100 - lowerBound

	var rbuf, wbuf bytes.Buffer

	_, err := io.Copy(&rbuf, in)
	if err != nil {
		return nil, err
	}

	for i := 0; i < cycles; i++ {
		fmt.Fprint(os.Stderr, ".")

		quality := lowerBound + rand.Intn(randRange)
		if err = recompress(&rbuf, &wbuf, quality); err != nil {
			return nil, err
		}
		// Prevent allocations by reseting and swapping
		rbuf.Reset()
		rbuf, wbuf = wbuf, rbuf
	}
	fmt.Fprintln(os.Stderr)
	return &rbuf, nil
}

func die(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage of shitpic:")
		fmt.Fprintln(os.Stderr, "\tshitpic [options] input output")
		flag.PrintDefaults()
	}

	cycles := flag.Uint("cycles", 100, "How many times to reprocess input")
	lowerBound := flag.Int("quality", 75, "Lower bound of quality (0–100)")

	flag.Parse()

	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	if *lowerBound < 0 || *lowerBound > 100 {
		flag.Usage()
		os.Exit(2)
	}

	var inf = os.Stdin
	if flag.Arg(0) != "-" {
		f, err := os.Open(flag.Arg(0))
		die(err)
		defer f.Close()
		inf = f
	}

	out, err := uglify(inf, int(*cycles), *lowerBound)
	die(err)

	var outf = os.Stdout
	if flag.Arg(1) != "-" {
		f, err := os.Create(flag.Arg(1))
		die(err)
		defer f.Close()
		outf = f
	}

	_, err = io.Copy(outf, out)
	die(err)
}
<commit_msg>Add reduce color flag<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"image"
	"image/gif"
	"image/jpeg"
	"io"
	"math/rand"
	"os"

	_ "image/png"
)

func recompress(in io.Reader, out io.Writer, quality int) error {
	img, _, err := image.Decode(in)
	if err != nil {
		return err
	}

	return jpeg.Encode(out, img, &jpeg.Options{Quality: quality})
}

func gifize(in io.Reader, out io.Writer) error {
	img, _, err := image.Decode(in)
	if err != nil {
		return err
	}

	return gif.Encode(out, img, nil)
}

func uglify(in io.Reader, cycles, lowerBound int) (io.Reader, error) {
	randRange := 100 - lowerBound

	var rbuf, wbuf bytes.Buffer

	_, err := io.Copy(&rbuf, in)
	if err != nil {
		return nil, err
	}

	for i := 0; i < cycles; i++ {
		fmt.Fprint(os.Stderr, ".")

		quality := lowerBound + rand.Intn(randRange)
		if err = recompress(&rbuf, &wbuf, quality); err != nil {
			return nil, err
		}
		// Prevent allocations by reseting and swapping
		rbuf.Reset()
		rbuf, wbuf = wbuf, rbuf
	}
	fmt.Fprintln(os.Stderr)
	return &rbuf, nil
}

func die(err error) {
	if err != nil {
		fmt.Fprintf(os.Stderr, "Error: %v\n", err)
		os.Exit(1)
	}
}

func main() {
	flag.Usage = func() {
		fmt.Fprintln(os.Stderr, "Usage of shitpic:")
		fmt.Fprintln(os.Stderr, "\tshitpic [options] input output")
		flag.PrintDefaults()
	}

	cycles := flag.Uint("cycles", 100, "How many times to reprocess input")
	lowerBound := flag.Int("quality", 75, "Lower bound of quality (0–100)")
	reduceColor := flag.Bool("reduce-colors", false, "Reduce to 256 colors")

	flag.Parse()

	if flag.NArg() != 2 {
		flag.Usage()
		os.Exit(2)
	}

	if *lowerBound < 0 || *lowerBound > 100 {
		flag.Usage()
		os.Exit(2)
	}

	var inf io.Reader = os.Stdin
	if flag.Arg(0) != "-" {
		f, err := os.Open(flag.Arg(0))
		die(err)
		defer f.Close()
		inf = f
	}

	if *reduceColor {
		var buf bytes.Buffer
		die(gifize(inf, &buf))
		inf = &buf
	}

	out, err := uglify(inf, int(*cycles), *lowerBound)
	die(err)

	var outf = os.Stdout
	if flag.Arg(1) != "-" {
		f, err := os.Create(flag.Arg(1))
		die(err)
		defer f.Close()
		outf = f
	}

	_, err = io.Copy(outf, out)
	die(err)
}

{'fraction_non_alphanumeric': 0.17306245297215953, 'fraction_numerical': 0.013293202909455732, 'mean_word_length': 3.251599147121535, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6757429', 'n_tokens_mistral': 1775, 'n_tokens_neox': 1631, 'n_words': 412}