<commit_before>package main

import (
	"errors"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

var (
	ErrUnknownFormat = errors.New("Image is in unknown format")
)

type Decoder struct {
	files   []string
	fileNum int
	image   image.Image
	y       int
}

func NewDecoder(path string) *Decoder {
	files, err := getFilenames(path)
	if err != nil {
		return nil
	}

	d := &Decoder{
		files:   files,
		fileNum: 0,
		image:   nil,
		y:       0,
	}

	if d.NextImage() {
		return d
	}

	return nil
}

func getFilenames(path string) ([]string, error) {
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return []string{}, err
	}

	o := []string{}
	for _, file := range files {
		n := file.Name()
		if strings.HasPrefix(n, ".") || strings.HasPrefix(n, "_") || file.IsDir() {
			continue
		}
		o = append(o, path+"/"+n)
	}

	return o, nil
}

func (d *Decoder) NextImage() bool {
	d.image = nil
	d.y = 0
	for {
		if len(d.files) == 0 {
			return false
		}

		if d.fileNum >= len(d.files) {
			d.fileNum = 0
		}
		file := d.files[d.fileNum]

		img, err := readImage(file)
		if err == nil {
			d.image = img
			d.y = img.Bounds().Min.Y
			d.fileNum++
			return true
		}

		log.Println("Error reading", file, err)
		d.files = append(d.files[:d.fileNum], d.files[d.fileNum+1:]...)
	}
}<commit_msg>Dont re-decode image if we only have one.<commit_after>package main

import (
	"errors"
	"image"
	_ "image/jpeg"
	_ "image/png"
	"io/ioutil"
	"log"
	"os"
	"strings"
)

var (
	ErrUnknownFormat = errors.New("Image is in unknown format")
)

type Decoder struct {
	files   []string
	fileNum int
	image   image.Image
	y       int
}

func NewDecoder(path string) *Decoder {
	files, err := getFilenames(path)
	if err != nil {
		return nil
	}

	d := &Decoder{
		files:   files,
		fileNum: 0,
		image:   nil,
		y:       0,
	}

	if d.NextImage() {
		return d
	}

	return nil
}

func getFilenames(path string) ([]string, error) {
	files, err := ioutil.ReadDir(path)
	if err != nil {
		return []string{}, err
	}

	o := []string{}
	for _, file := range files {
		n := file.Name()
		if strings.HasPrefix(n, ".") || strings.HasPrefix(n, "_") || file.IsDir() {
			continue
		}
		o = append(o, path+"/"+n)
	}

	return o, nil
}

func (d *Decoder) NextImage() bool {
	d.y = 0

	// If we only have one image and it's already loaded, we're done.
	if d.image != nil && len(d.files) < 2 {
		return true
	}

	d.image = nil
	for {
		if len(d.files) == 0 {
			return false
		}

		if d.fileNum >= len(d.files) {
			d.fileNum = 0
		}
		file := d.files[d.fileNum]

		img, err := readImage(file)
		if err == nil {
			d.image = img
			d.y = img.Bounds().Min.Y
			d.fileNum++
			return true
		}

		log.Println("Error reading", file, err)
		d.files = append(d.files[:d.fileNum], d.files[d.fileNum+1:]...)
	}
}
{'fraction_non_alphanumeric': 0.1669646049338577, 'fraction_numerical': 0.004647836968180193, 'mean_word_length': 2.6961690885072653, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5055995', 'n_tokens_mistral': 1243, 'n_tokens_neox': 1137, 'n_words': 290}