<commit_before>package main

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/cheggaaa/pb"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"math"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"strings"
)

var subdivisions *int
var tolerance *int

var scratchDir string

type pictable [][][]uint64

func MkPictable(dx int, dy int) pictable {
	pic := make([][][]uint64, dx) /* type declaration */
	for i := range pic {
		pic[i] = make([][]uint64, dy) /* again the type? */
		for j := range pic[i] {
			pic[i][j] = []uint64{0, 0, 0}
		}
	}
	return pic
}

func absdiff(a uint64, b uint64) uint64 {
	return uint64(math.Abs(float64(a) - float64(b)))
}

func init() {
	subdivisions = flag.Int("subdivisions", 10, "Slices per axis")
	tolerance = flag.Int("tolerance", 100, "Color delta tolerance, higher = more tolerant")
	flag.Parse()

	if flag.NArg() < 1 {
		fmt.Println("usage: imgdedup [options] [<directories>/files]")
		flag.PrintDefaults()
		os.Exit(1)
	}

	image.RegisterFormat("png", "png", png.Decode, png.DecodeConfig)
	image.RegisterFormat("jpeg", "jpeg", jpeg.Decode, jpeg.DecodeConfig)
	image.RegisterFormat("gif", "gif", gif.Decode, gif.DecodeConfig)
}

func init() {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	scratchDir = path.Join(usr.HomeDir, ".imgdedup")

	if _, err := os.Stat(scratchDir); err != nil {
		if os.IsNotExist(err) {
			os.Mkdir(scratchDir, 0700)
		} else {
			log.Fatal(err)
		}
	}
}

func scanImg(file *os.File) (pictable, error) {
	m, _, err := image.Decode(file)
	if err != nil {
		return nil, err
	}
	bounds := m.Bounds()

	avgdata := MkPictable(*subdivisions, *subdivisions)

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			rX := int64(math.Floor((float64(x) / float64(bounds.Max.X)) * float64(*subdivisions)))
			rY := int64(math.Floor((float64(y) / float64(bounds.Max.Y)) * float64(*subdivisions)))

			r, g, b, _ := m.At(x, y).RGBA()
			avgdata[rX][rY][0] += uint64((float32(r) / 65535) * 255)
			avgdata[rX][rY][1] += uint64((float32(g) / 65535) * 255)
			avgdata[rX][rY][2] += uint64((float32(b) / 65535) * 255)
		}
	}

	divisor := uint64((bounds.Max.X / *subdivisions) * (bounds.Max.Y / *subdivisions))

	for rX := 0; rX < *subdivisions; rX++ {
		for rY := 0; rY < *subdivisions; rY++ {
			avgdata[rX][rY][0] = avgdata[rX][rY][0] / divisor
			avgdata[rX][rY][1] = avgdata[rX][rY][1] / divisor
			avgdata[rX][rY][2] = avgdata[rX][rY][2] / divisor
		}
	}

	return avgdata, nil
}

func loadCache(cachename string) (pictable, error) {

	file, err := os.Open(cachename)
	if err != nil {
		return nil, err
	}
	r := bufio.NewReader(file)

	var avgdata pictable

	dec := json.NewDecoder(r)

	err = dec.Decode(&avgdata)
	if err != nil {
		return nil, err
	}

	return avgdata, nil
}

func storeCache(cachename string, avgdata *pictable) {
	fo, err := os.Create(cachename)
	if err != nil {
		panic(err)
	}

	enc := json.NewEncoder(fo)
	enc.Encode(avgdata)
	fo.Close()
}

func main() {
	
	imgdata := make(map[string]pictable)

	fileList := getFiles(flag.Args())

	bar := pb.StartNew(len(fileList))

	for _, imgpath := range fileList {

		bar.Increment()

		file, err := os.Open(imgpath)
		if err != nil {
			log.Fatal(err)
		}

		fExt := strings.ToLower(filepath.Ext(imgpath))
		if fExt == ".png" || fExt == ".jpg" || fExt == ".jpeg" || fExt == ".gif" {

			fi, err := file.Stat()
			if err != nil {
				log.Fatal(err)<commit_msg>Remember to close cache files<commit_after>package main

import (
	"bufio"
	"crypto/md5"
	"encoding/json"
	"flag"
	"fmt"
	"github.com/cheggaaa/pb"
	"image"
	"image/gif"
	"image/jpeg"
	"image/png"
	"io"
	"log"
	"math"
	"os"
	"os/user"
	"path"
	"path/filepath"
	"strings"
)

var subdivisions *int
var tolerance *int

var scratchDir string

type pictable [][][]uint64

func MkPictable(dx int, dy int) pictable {
	pic := make([][][]uint64, dx) /* type declaration */
	for i := range pic {
		pic[i] = make([][]uint64, dy) /* again the type? */
		for j := range pic[i] {
			pic[i][j] = []uint64{0, 0, 0}
		}
	}
	return pic
}

func absdiff(a uint64, b uint64) uint64 {
	return uint64(math.Abs(float64(a) - float64(b)))
}

func init() {
	subdivisions = flag.Int("subdivisions", 10, "Slices per axis")
	tolerance = flag.Int("tolerance", 100, "Color delta tolerance, higher = more tolerant")
	flag.Parse()

	if flag.NArg() < 1 {
		fmt.Println("usage: imgdedup [options] [<directories>/files]")
		flag.PrintDefaults()
		os.Exit(1)
	}

	image.RegisterFormat("png", "png", png.Decode, png.DecodeConfig)
	image.RegisterFormat("jpeg", "jpeg", jpeg.Decode, jpeg.DecodeConfig)
	image.RegisterFormat("gif", "gif", gif.Decode, gif.DecodeConfig)
}

func init() {
	usr, err := user.Current()
	if err != nil {
		log.Fatal(err)
	}
	scratchDir = path.Join(usr.HomeDir, ".imgdedup")

	if _, err := os.Stat(scratchDir); err != nil {
		if os.IsNotExist(err) {
			os.Mkdir(scratchDir, 0700)
		} else {
			log.Fatal(err)
		}
	}
}

func scanImg(file *os.File) (pictable, error) {
	m, _, err := image.Decode(file)
	if err != nil {
		return nil, err
	}
	bounds := m.Bounds()

	avgdata := MkPictable(*subdivisions, *subdivisions)

	for y := bounds.Min.Y; y < bounds.Max.Y; y++ {
		for x := bounds.Min.X; x < bounds.Max.X; x++ {
			rX := int64(math.Floor((float64(x) / float64(bounds.Max.X)) * float64(*subdivisions)))
			rY := int64(math.Floor((float64(y) / float64(bounds.Max.Y)) * float64(*subdivisions)))

			r, g, b, _ := m.At(x, y).RGBA()
			avgdata[rX][rY][0] += uint64((float32(r) / 65535) * 255)
			avgdata[rX][rY][1] += uint64((float32(g) / 65535) * 255)
			avgdata[rX][rY][2] += uint64((float32(b) / 65535) * 255)
		}
	}

	divisor := uint64((bounds.Max.X / *subdivisions) * (bounds.Max.Y / *subdivisions))

	for rX := 0; rX < *subdivisions; rX++ {
		for rY := 0; rY < *subdivisions; rY++ {
			avgdata[rX][rY][0] = avgdata[rX][rY][0] / divisor
			avgdata[rX][rY][1] = avgdata[rX][rY][1] / divisor
			avgdata[rX][rY][2] = avgdata[rX][rY][2] / divisor
		}
	}

	return avgdata, nil
}

func loadCache(cachename string) (pictable, error) {

	file, err := os.Open(cachename)
	defer file.Close()
	if err != nil {
		return nil, err
	}
	r := bufio.NewReader(file)

	var avgdata pictable

	dec := json.NewDecoder(r)

	err = dec.Decode(&avgdata)
	if err != nil {
		return nil, err
	}

	return avgdata, nil
}

func storeCache(cachename string, avgdata *pictable) {
	fo, err := os.Create(cachename)
	defer fo.Close()
	if err != nil {
		panic(err)
	}

	enc := json.NewEncoder(fo)
	enc.Encode(avgdata)
}

func main() {
	
	imgdata := make(map[string]pictable)

	fileList := getFiles(flag.Args())

	bar := pb.StartNew(len(fileList))

	for _, imgpath := range fileList {

		bar.Increment()

		file, err := os.Open(imgpath)
		if err != nil {
			log.Fatal(err)
		}

		fExt := strings.ToLower(filepath.Ext(imgpath))
		if fExt == ".png" || fExt == ".jpg" || fExt == ".jpeg" || fExt == ".gif" {

			fi, err := file.Stat()
			if err != nil {
				log.Fatal(err)
{'fraction_non_alphanumeric': 0.19177885992552277, 'fraction_numerical': 0.028645087367516472, 'mean_word_length': 3.6865771812080537, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1241585', 'n_tokens_mistral': 3234, 'n_tokens_neox': 2919, 'n_words': 618}