<commit_before>package dhash

import (
	"fmt"
	"image"

	"github.com/andybalholm/go-bit"
)

// A Hash is a 128-bit perceptual hash.
type Hash [2]uint64

func (h Hash) String() string {
	return fmt.Sprintf("%016x%016x", h[0], h[1])
}

// New returns a hash of img. The algorithm is the difference hash from
// http://www.hackerfactor.com/blog/index.php?/archives/529-Kind-of-Like-That.html.
func New(img image.Image) Hash {
	bounds := img.Bounds()
	width := bounds.Max.X - bounds.Min.X
	height := bounds.Max.Y - bounds.Min.Y

	// Calculate the mean brightness of each block in an 9x9 grid.
	var blocks [9][9]int
	for i := 0; i < 9; i++ {
		left := bounds.Min.X + (width * i / 9)
		right := bounds.Min.X + (width * (i + 1) / 9)
		if right == left {
			right = left + 1
		}
		for j := 0; j < 9; j++ {
			top := bounds.Min.Y + (height * j / 9)<commit_msg>Add Parse function.<commit_after>package dhash

import (
	"fmt"
	"image"
	"strings"

	"github.com/andybalholm/go-bit"
)

// A Hash is a 128-bit perceptual hash.
type Hash [2]uint64

func (h Hash) String() string {
	return fmt.Sprintf("%016x%016x", h[0], h[1])
}

// Parse takes the string representation of a Hash, and returns the
// corresponding Hash value.
func Parse(s string) (h Hash, err error) {
	if len(s) != 32 {
		err = fmt.Errorf("wrong length for dhash value (%d characters; should be 32)", len(s))
		return
	}
	_, err = fmt.Fscanf(strings.NewReader(s), "%016x%016x", &h[0], &h[1])
	return
}

// New returns a hash of img. The algorithm is the difference hash from
// http://www.hackerfactor.com/blog/index.php?/archives/529-Kind-of-Like-That.html.
func New(img image.Image) Hash {
	bounds := img.Bounds()
	width := bounds.Max.X - bounds.Min.X
	height := bounds.Max.Y - bounds.Min.Y

	// Calculate the mean brightness of each block in an 9x9 grid.
	var blocks [9][9]int
	for i := 0; i < 9; i++ {
		left := bounds.Min.X + (width * i / 9)
		right := bounds.Min.X + (width * (i + 1) / 9)
		if right == left {
			right = left + 1
		}
		for j := 0; j < 9; j++ {
			top := bounds.Min.Y + (height * j / 9)
{'fraction_non_alphanumeric': 0.16690926734594858, 'fraction_numerical': 0.034934497816593885, 'mean_word_length': 2.9806949806949805, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6352920', 'n_tokens_mistral': 855, 'n_tokens_neox': 772, 'n_words': 243}