<commit_before>package indexer

import (
	"errors"
	"fmt"
	"io"
	"net/http"

	"github.com/cardigann/cardigann/torznab"
	"golang.org/x/sync/errgroup"
)

type Aggregate []torznab.Indexer

func (ag Aggregate) Search(query torznab.Query) ([]torznab.ResultItem, error) {
	g := errgroup.Group{}
	allResults := make([][]torznab.ResultItem, len(ag))
	maxLength := 0

	// fetch all results
	for idx, indexer := range ag {
		indexerID := indexer.Info().ID
		idx, indexer := idx, indexer
		g.Go(func() error {
			result, err := indexer.Search(query)
			if err != nil {
				return fmt.Errorf("Indexer %q failed: %s", indexerID, err)
			}
			allResults[idx] = result
			if l := len(result); l > maxLength {
				maxLength = l
			}
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		return nil, err
	}

	results := []torznab.ResultItem{}

	// interleave search results to preserve ordering
	for i := 0; i <= maxLength; i++ {
		for _, row := range allResults {
			if len(results) > i {
				results = append(results, row[i])<commit_msg>Aggregate working<commit_after>package indexer

import (
	"errors"
	"io"
	"net/http"

	"github.com/cardigann/cardigann/torznab"
	"golang.org/x/sync/errgroup"
)

type Aggregate []torznab.Indexer

func (ag Aggregate) Search(query torznab.Query) ([]torznab.ResultItem, error) {
	g := errgroup.Group{}
	allResults := make([][]torznab.ResultItem, len(ag))
	maxLength := 0

	// fetch all results
	for idx, indexer := range ag {
		indexerID := indexer.Info().ID
		idx, indexer := idx, indexer
		g.Go(func() error {
			result, err := indexer.Search(query)
			if err != nil {
				log.Warnf("Indexer %q failed: %s", indexerID, err)
				return nil
			}
			allResults[idx] = result
			if l := len(result); l > maxLength {
				maxLength = l
			}
			return nil
		})
	}
	if err := g.Wait(); err != nil {
		log.Warn(err)
		return nil, err
	}

	results := []torznab.ResultItem{}

	// interleave search results to preserve ordering
	for i := 0; i <= maxLength; i++ {
		for _, r := range allResults {
			if len(r) > i {
				results = append(results, r[i])
{'fraction_non_alphanumeric': 0.15470417070805043, 'fraction_numerical': 0.0019398642095053346, 'mean_word_length': 3.343157894736842, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4471178', 'n_tokens_mistral': 846, 'n_tokens_neox': 766, 'n_words': 197}