<commit_before>package internal

import (
	"github.com/101loops/hrd/internal/trafo"
	"github.com/101loops/hrd/internal/types"

	ds "appengine/datastore"
)

// Iterate loads entities from an iterator.
func Iterate(dsIt *ds.Iterator, dsts interface{}, multi bool) (keys []*types.Key, err error) {

	// in a keys-only query there is no dsts
	var docSet *trafo.DocList
	if dsts != nil {
		docSet, err = trafo.NewWriteableDocList(dsts, nil, multi)
		if err != nil {
			return
		}
	}

	var dsDocs []*trafo.Doc
	var dsKeys []*ds.Key
	for i := 0; ; i++ {

		// prepare next doc
		var doc *trafo.Doc
		if docSet != nil {
			doc, err = docSet.Get(i)
			if err != nil {
				return
			}
			dsDocs = append(dsDocs, doc)
		}

		var dsKey *ds.Key
		dsKey, err = dsIt.Next(doc)
		if err == ds.Done {
			if !multi {
				if doc != nil {
					doc.Nil()
				}
				return nil, nil
			}
			break
		}
		if err != nil {
			return
		}

		dsKeys = append(dsKeys, dsKey)

		if !multi {
			break
		}
	}

	keys, err = applyResult(dsDocs, dsKeys, err)
	if dsDocs != nil {
		for i := range keys {
			docSet.Add(keys[i], dsDocs[i])
		}
	}

	return keys, err
}<commit_msg>rename docSet to docList<commit_after>package internal

import (
	"github.com/101loops/hrd/internal/trafo"
	"github.com/101loops/hrd/internal/types"

	ds "appengine/datastore"
)

// Iterate loads entities from an iterator.
func Iterate(dsIt *ds.Iterator, dsts interface{}, multi bool) (keys []*types.Key, err error) {

	// in a keys-only query there is no dsts
	var docList *trafo.DocList
	if dsts != nil {
		docList, err = trafo.NewWriteableDocList(dsts, nil, multi)
		if err != nil {
			return
		}
	}

	var dsDocs []*trafo.Doc
	var dsKeys []*ds.Key
	for i := 0; ; i++ {

		// prepare next doc
		var doc *trafo.Doc
		if docList != nil {
			doc, err = docList.Get(i)
			if err != nil {
				return
			}
			dsDocs = append(dsDocs, doc)
		}

		var dsKey *ds.Key
		dsKey, err = dsIt.Next(doc)
		if err == ds.Done {
			if !multi {
				if doc != nil {
					doc.Nil()
				}
				return nil, nil
			}
			break
		}
		if err != nil {
			return
		}

		dsKeys = append(dsKeys, dsKey)

		if !multi {
			break
		}
	}

	keys, err = applyResult(dsDocs, dsKeys, err)
	if dsDocs != nil {
		for i := range keys {
			docList.Add(keys[i], dsDocs[i])
		}
	}

	return keys, err
}
{'fraction_non_alphanumeric': 0.13600697471665213, 'fraction_numerical': 0.006102877070619006, 'mean_word_length': 2.59717868338558, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1827677', 'n_tokens_mistral': 1052, 'n_tokens_neox': 911, 'n_words': 247}