<commit_before>package hammer

import (
	"context"
	"log"

	"github.com/antihax/evedata/internal/datapackages"
	"github.com/antihax/goesi"
	"github.com/antihax/goesi/esi"
)

func init() {
	registerConsumer("structure", structureConsumer)
	registerConsumer("structureOrders", structureOrdersConsumer)
}

func structureConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, s.token)
	struc, _, err := s.esi.ESI.UniverseApi.GetUniverseStructuresStructureId(ctx, structureID, nil)
	if err != nil {
		s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 86400)
		return
	}

	// Send out the result
	err = s.QueueResult(&datapackages.Structure{Structure: struc, StructureID: structureID}, "structure")
	if err != nil {
		log.Println(err)
		return
	}
}

func structureOrdersConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)
	var page int32 = 1
	orders := []esi.GetMarketsStructuresStructureId200Ok{}

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, s.token)

	for {
		o, _, err := s.esi.ESI.MarketApi.GetMarketsStructuresStructureId(ctx, structureID, map[string]interface{}{"page": page})
		if err != nil {
			s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 172800)
			return
		} else if len(o) == 0 { // end of the pages
			break
		}
		orders = append(orders, o...)

		page++
	}

	// early out if there are no orders
	if len(orders) == 0 {
		return
	}

	// Send out the result
	err := s.QueueResult(&datapackages.StructureOrders{Orders: orders, StructureID: structureID}, "structureOrders")
	if err != nil {
		log.Println(err)
		return
	}
}
<commit_msg>get a bit more chatty about bad structures<commit_after>package hammer

import (
	"context"
	"log"

	"github.com/antihax/evedata/internal/datapackages"
	"github.com/antihax/goesi"
	"github.com/antihax/goesi/esi"
)

func init() {
	registerConsumer("structure", structureConsumer)
	registerConsumer("structureOrders", structureOrdersConsumer)
}

func structureConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, s.token)
	struc, _, err := s.esi.ESI.UniverseApi.GetUniverseStructuresStructureId(ctx, structureID, nil)
	if err != nil {
		log.Printf("Bad structure: %s %d\n", err, structureID)
		err := s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 86400)
		if err != nil {
			log.Printf("failed setting failure: %s %d\n", err, structureID)
		}
		return
	}

	// Send out the result
	err = s.QueueResult(&datapackages.Structure{Structure: struc, StructureID: structureID}, "structure")
	if err != nil {
		log.Println(err)
		return
	}
}

func structureOrdersConsumer(s *Hammer, parameter interface{}) {
	structureID := parameter.(int64)
	var page int32 = 1
	orders := []esi.GetMarketsStructuresStructureId200Ok{}

	if s.inQueue.CheckWorkExpired("evedata_structure_failure", structureID) {
		return
	}

	ctx := context.WithValue(context.Background(), goesi.ContextOAuth2, s.token)

	for {
		o, _, err := s.esi.ESI.MarketApi.GetMarketsStructuresStructureId(ctx, structureID, map[string]interface{}{"page": page})
		if err != nil {
			log.Printf("Bad structure: %s %d\n", err, structureID)
			err := s.inQueue.SetWorkExpire("evedata_structure_failure", structureID, 86400)
			if err != nil {
				log.Printf("failed setting failure: %s %d\n", err, structureID)
			}
			return
		} else if len(o) == 0 { // end of the pages
			break
		}
		orders = append(orders, o...)

		page++
	}

	// early out if there are no orders
	if len(orders) == 0 {
		return
	}

	// Send out the result
	err := s.QueueResult(&datapackages.StructureOrders{Orders: orders, StructureID: structureID}, "structureOrders")
	if err != nil {
		log.Println(err)
		return
	}
}

{'fraction_non_alphanumeric': 0.13352685050798258, 'fraction_numerical': 0.01185292694726657, 'mean_word_length': 4.727146814404432, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1527036', 'n_tokens_mistral': 1530, 'n_tokens_neox': 1417, 'n_words': 301}