<commit_before>package crawler

import (
	"context"
	"fmt"
	"log"
	"time"

	"go.opentelemetry.io/otel/label"

	index_types "github.com/ipfs-search/ipfs-search/components/index/types"
	t "github.com/ipfs-search/ipfs-search/types"
)

func appendReference(refs index_types.References, r *t.Reference) (index_types.References, bool) {
	if r.Parent == nil {
		// No new reference, not updating
		return refs, false
	}

	for _, indexedRef := range refs {
		if indexedRef.ParentHash == r.Parent.ID && indexedRef.Name == r.Name {
			// Existing reference, not updating
			return refs, false
		}
	}

	return append(refs, index_types.Reference{
		ParentHash: r.Parent.ID,
		Name:       r.Name,
	}), true
}

// updateExisting updates known existing items.
func (c *Crawler) updateExisting(ctx context.Context, i *existingItem) error {
	ctx, span := c.Tracer.Start(ctx, "crawler.updateExisting")
	defer span.End()

	switch i.Source {
	case t.DirectorySource:
		// Item referenced from a directory, consider updating references (but not last-seen).
		refs, refsUpdated := appendReference(i.References, &i.AnnotatedResource.Reference)

		if refsUpdated {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "reference-added"),
				label.Any("new-reference", i.AnnotatedResource.Reference),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				References: refs,
			})
		}

	case t.SnifferSource, t.UnknownSource:
		// TODO: Remove UnknownSource after sniffer is updated and queue is flushed.
		// Item sniffed, conditionally update last-seen.
		now := time.Now()

		// Strip milliseconds to cater to legacy ES index format.
		// This can be safely removed after the next reindex with _nomillis removed from time format.
		now = now.Truncate(time.Second)

		var isRecent bool
		if i.LastSeen == nil {
			// No LastSeen set, override isRecent
			isRecent = true
		} else {
			isRecent = now.Sub(*i.LastSeen) > c.config.MinUpdateAge
		}

		if isRecent {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "is-recent"),
				label.Stringer("last-seen", i.LastSeen),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				LastSeen: &now,
			})
		}

	case t.ManualSource, t.UserSource:
		// Do not update based on manual or user input.

	default:
		// Panic for unexpected Source values, instead of hard failing.
		panic(fmt.Sprintf("Unexpected source %s for item %+v", i.Source, i))
	}

	span.AddEvent(ctx, "Not updating")

	return nil
}
<commit_msg>Fixup for previous patch.<commit_after>package crawler

import (
	"context"
	"fmt"
	"log"
	"time"

	"go.opentelemetry.io/otel/label"

	index_types "github.com/ipfs-search/ipfs-search/components/index/types"
	t "github.com/ipfs-search/ipfs-search/types"
)

func appendReference(refs index_types.References, r *t.Reference) (index_types.References, bool) {
	if r.Parent == nil {
		// No new reference, not updating
		return refs, false
	}

	for _, indexedRef := range refs {
		if indexedRef.ParentHash == r.Parent.ID && indexedRef.Name == r.Name {
			// Existing reference, not updating
			return refs, false
		}
	}

	return append(refs, index_types.Reference{
		ParentHash: r.Parent.ID,
		Name:       r.Name,
	}), true
}

// updateExisting updates known existing items.
func (c *Crawler) updateExisting(ctx context.Context, i *existingItem) error {
	ctx, span := c.Tracer.Start(ctx, "crawler.updateExisting")
	defer span.End()

	switch i.Source {
	case t.DirectorySource:
		// Item referenced from a directory, consider updating references (but not last-seen).
		refs, refsUpdated := appendReference(i.References, &i.AnnotatedResource.Reference)

		if refsUpdated {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "reference-added"),
				label.Any("new-reference", i.AnnotatedResource.Reference),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				References: refs,
			})
		}

	case t.SnifferSource, t.UnknownSource:
		// TODO: Remove UnknownSource after sniffer is updated and queue is flushed.
		// Item sniffed, conditionally update last-seen.
		now := time.Now()

		// Strip milliseconds to cater to legacy ES index format.
		// This can be safely removed after the next reindex with _nomillis removed from time format.
		now = now.Truncate(time.Second)

		var isRecent bool
		if i.LastSeen == nil {
			// No LastSeen set, override isRecent
			isRecent = true
		} else {
			isRecent = now.Sub(*i.LastSeen) > c.config.MinUpdateAge
		}

		if isRecent {
			span.AddEvent(ctx, "Updating",
				label.String("reason", "is-recent"),
				// TODO: This causes a panic when LastSeen is nil.
				// label.Stringer("last-seen", i.LastSeen),
			)

			return i.Index.Update(ctx, i.AnnotatedResource.ID, &index_types.Update{
				LastSeen: &now,
			})
		}

	case t.ManualSource, t.UserSource:
		// Do not update based on manual or user input.

	default:
		// Panic for unexpected Source values, instead of hard failing.
		panic(fmt.Sprintf("Unexpected source %s for item %+v", i.Source, i))
	}

	span.AddEvent(ctx, "Not updating")

	return nil
}

{'fraction_non_alphanumeric': 0.12278978388998035, 'fraction_numerical': 0.0, 'mean_word_length': 3.582358235823582, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7631449', 'n_tokens_mistral': 1901, 'n_tokens_neox': 1683, 'n_words': 450}