<commit_before>package fetcheddit

import (
	"code.leeclagett.com/grokeddit"
	"errors"
)

type nextChunk struct {
	nextThings     []grokeddit.Thing
	moreThings     bool
	retrievalError error
}

type thingList struct {
	fetchMoreThings func() ([]grokeddit.Thing, bool, error)
	currentThings   thingIterater // Dictates how a slice returned by fetchMoreThings is iterated
	moreThings      bool          // The last value returned by fetchMoreThings
	chunkChannel    chan nextChunk
}

func fetchThingList(
	fetchGroked func(Fetcher, *AnchorPoint) (grokeddit.Groked, error),
	contentFetcher Fetcher,
	anchor *AnchorPoint) (thingList, error) {

	var currentThingIterater thingIterater
	var fetchMore func() ([]grokeddit.Thing, bool, error)

	// make a copy of the anchor, since it will be stored in a
	// closure thats invoked asynchronously
	if anchor != nil {
		newAnchor := new(AnchorPoint)
		*newAnchor = *anchor
		anchor = newAnchor
	}

	if anchor == nil || anchor.Direction == Next {
		currentThingIterater = &forwardThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(contentFetcher, anchor)
			if error != nil {
				return nil, false, error
			}

			if anchor == nil {
				anchor = new(AnchorPoint)
				anchor.Direction = Next
			}

			if newGroked.ListingNext != nil {
				anchor.Anchor = *newGroked.ListingNext
			}

			return newGroked.Children, newGroked.ListingNext != nil, nil
		}
	} else {
		currentThingIterater = &reverseThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(contentFetcher, anchor)
			if error != nil {
				return nil, false, error
			}

			// There is a bug on the reddit side, the "before"
			// field is always NULL with a before request. This is
			// solved with a hack that will do 1 extra request
			// than necessary, the additional traffic is on you
			// reddit.
			more := false
			if newGroked.Children != nil && len(newGroked.Children) > 0 {
				anchor.Anchor = newGroked.Children[0].Id
				more = true
			}

			return newGroked.Children, more, nil
		}
	}

	firstThings, moreThings, error := fetchMore()
	if error != nil {
		return thingList{}, errors.New("No things in first request")
	}

	currentThingIterater.setArray(firstThings)
	newThingList := thingList{fetchMore, currentThingIterater, moreThings, make(chan nextChunk)}

	if moreThings {
		newThingList.fetchNextBlockAsync()
	}
	return newThingList, nil
}

func (list *thingList) fetchNextBlockAsync() {
	go func() {
		nextThings, moreThings, error := list.fetchMoreThings()
		list.chunkChannel <- nextChunk{nextThings, moreThings, error}
	}()
}

func (list *thingList) hasNext() bool {
	return list.moreThings || list.currentThings.hasNext()
}

func (list *thingList) getNext() (grokeddit.Thing, error) {
	if !list.hasNext() {<commit_msg>Finished comment thingList<commit_after>package fetcheddit

import (
	"code.leeclagett.com/grokeddit"
	"errors"
)

// Aggregate returned in the channel that fetches reddit thing lists
type nextChunk struct {
	nextThings     []grokeddit.Thing // The list of things retrieved
	moreThings     bool              // True if there is another list of things
	retrievalError error             // Non-nil if there was an error trying to retrieve the list of things
}

/* An abstraction for a list of reddit things. The paginated lists are
"flattened" to appear as giant single list. The pages are retreived
asynchronously, so there is less blocking time waiting for the next page. */
type thingList struct {
	fetchMoreThings func() ([]grokeddit.Thing, bool, error) // Function that can be called in a separate goroutine to retrieve the next page of things
	currentThings   thingIterater                           // Dictates how a slice returned by fetchMoreThings is iterated
	moreThings      bool                                    // The last value returned by fetchMoreThings
	chunkChannel    chan nextChunk                          // Receives the next page of reddit things
}

/* Creates a new thing list. 

The closure arugment (fetchedGroked) allows the caller to handle differences in
the reddit API. The lists can be sent in two different ways depending on the 
context, and the caller must massage those differences into a single Groked 
object.

The contentFetcher allows the caller to modify how the reddit things are being 
retrieved, mainly used for testing (possibly different URLs too).

The anchor specifies a starting point, and an iteration direction. NULL 
indicates to start from the beginning, and iterate forward. */
func fetchThingList(
	fetchGroked func(Fetcher, *AnchorPoint) (grokeddit.Groked, error),
	contentFetcher Fetcher,
	anchor *AnchorPoint) (thingList, error) {

	var currentThingIterater thingIterater
	var fetchMore func() ([]grokeddit.Thing, bool, error)

	// make a copy of the anchor, since it will be stored in a
	// closure thats invoked asynchronously
	if anchor != nil {
		newAnchor := new(AnchorPoint)
		*newAnchor = *anchor
		anchor = newAnchor
	}

	if anchor == nil || anchor.Direction == Next {
		currentThingIterater = &forwardThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(contentFetcher, anchor)
			if error != nil {
				return nil, false, error
			}

			if anchor == nil {
				anchor = new(AnchorPoint)
				anchor.Direction = Next
			}

			if newGroked.ListingNext != nil {
				anchor.Anchor = *newGroked.ListingNext
			}

			return newGroked.Children, newGroked.ListingNext != nil, nil
		}
	} else {
		currentThingIterater = &reverseThingIterate{}
		fetchMore = func() ([]grokeddit.Thing, bool, error) {
			newGroked, error := fetchGroked(contentFetcher, anchor)
			if error != nil {
				return nil, false, error
			}

			// There is a bug on the reddit side, the "before"
			// field is always NULL with a before request. This is
			// solved with a hack that will do 1 extra request
			// than necessary, the additional traffic is on you
			// reddit.
			more := false
			if newGroked.Children != nil && len(newGroked.Children) > 0 {
				anchor.Anchor = newGroked.Children[0].Id
				more = true
			}

			return newGroked.Children, more, nil
		}
	}

	firstThings, moreThings, error := fetchMore()
	if error != nil {
		return thingList{}, errors.New("No things in first request")
	}

	currentThingIterater.setArray(firstThings)
	newThingList := thingList{fetchMore, currentThingIterater, moreThings, make(chan nextChunk)}

	if moreThings {
		newThingList.fetchNextBlockAsync()
	}
	return newThingList, nil
}

// Asynchronously fetches the next page of reddit things.
func (list *thingList) fetchNextBlockAsync() {
	go func() {
		nextThings, moreThings, error := list.fetchMoreThings()
		list.chunkChannel <- nextChunk{nextThings, moreThings, error}
	}()
}

// Return true if the thingList has another thing available.
func (list *thingList) hasNext() bool {
	return list.moreThings || list.currentThings.hasNext()
}

// Return the next thing in the list, or error.
func (list *thingList) getNext() (grokeddit.Thing, error) {
	if !list.hasNext() {
{'fraction_non_alphanumeric': 0.09119097956307258, 'fraction_numerical': 0.0008456659619450317, 'mean_word_length': 3.5988334413480234, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1424148', 'n_tokens_mistral': 2371, 'n_tokens_neox': 2129, 'n_words': 732}