<commit_before>package crawler

import (
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"net"
	"net/url"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index invalid to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			log.Printf("Temporary URL error: %v", uerr)
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// hashURL returns the IPFS URL for a particular hash
func (i *Indexable) hashURL() string {
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

// getFileList return list of files and/or type of item (directory/file)
func (i *Indexable) getFileList(ctx context.Context) (list *shell.UnixLsObject, err error) {
	url := i.hashURL()

	tryAgain := true
	for tryAgain {
		list, err = i.Shell.FileList(url)

		tryAgain, err = i.handleShellError(ctx, err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// indexInvalid indexes invalid files to prevent indexing again
func (i *Indexable) indexInvalid(ctx context.Context, err error) {
	// Attempt to index panic to prevent re-indexing
	m := metadata{
		"error": err.Error(),
	}

	i.Indexer.IndexItem(ctx, "invalid", i.Hash, m)
}

// queueList queues any items in a given list/directory
func (i *Indexable) queueList(ctx context.Context, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: i.Hash,
		}

		switch link.Type {
		case "File":
			// Add file to crawl queue, with lower priority
			err = i.FileQueue.Publish(dirArgs, 8)
		case "Directory":
			// Add directory to crawl queue, with lower priority
			err = i.HashQueue.Publish(dirArgs, 8)
		default:
			log.Printf("Type '%s' skipped for %s", link.Type, i)
			i.indexInvalid(ctx, fmt.Errorf("Unknown type: %s", link.Type))
		}
	}

	return
}

// processList processes and indexes a file listing
func (i *Indexable) processList(ctx context.Context, list *shell.UnixLsObject, references []indexer.Reference) (err error) {
	now := nowISO()

	switch list.Type {
	case "File":
		// Add to file crawl queue with high priority
		fileArgs := &Args{
			Hash:       i.Hash,
			Name:       i.Name,<commit_msg>Randomized priorities for directory entries.<commit_after>package crawler

import (
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"math/rand"
	"net"
	"net/url"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index invalid to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			log.Printf("Temporary URL error: %v", uerr)
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
		}
	}

	return false, err
}

// hashURL returns the IPFS URL for a particular hash
func (i *Indexable) hashURL() string {
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

// getFileList return list of files and/or type of item (directory/file)
func (i *Indexable) getFileList(ctx context.Context) (list *shell.UnixLsObject, err error) {
	url := i.hashURL()

	tryAgain := true
	for tryAgain {
		list, err = i.Shell.FileList(url)

		tryAgain, err = i.handleShellError(ctx, err)

		if tryAgain {
			log.Printf("Retrying in %s", i.Config.RetryWait)
			time.Sleep(i.Config.RetryWait)
		}
	}

	return
}

// indexInvalid indexes invalid files to prevent indexing again
func (i *Indexable) indexInvalid(ctx context.Context, err error) {
	// Attempt to index panic to prevent re-indexing
	m := metadata{
		"error": err.Error(),
	}

	i.Indexer.IndexItem(ctx, "invalid", i.Hash, m)
}

// queueList queues any items in a given list/directory
func (i *Indexable) queueList(ctx context.Context, list *shell.UnixLsObject) (err error) {
	for _, link := range list.Links {
		dirArgs := &Args{
			Hash:       link.Hash,
			Name:       link.Name,
			Size:       link.Size,
			ParentHash: i.Hash,
		}

		// Generate random lower priority for items in this directory
		// Rationale; directories might have different availability but
		// within a directory, items are likely to have similar availability.
		// We want consumers to get a varied mixture of availability, for
		// consistent overall indexing load.

		priority := uint8(rand.Intn(8))

		switch link.Type {
		case "File":
			// Add file to crawl queue, with lower priority
			err = i.FileQueue.Publish(dirArgs, priority)
		case "Directory":
			// Add directory to crawl queue, with lower priority
			err = i.HashQueue.Publish(dirArgs, priority)
		default:
			log.Printf("Type '%s' skipped for %s", link.Type, i)
			i.indexInvalid(ctx, fmt.Errorf("Unknown type: %s", link.Type))
		}
	}

	return
}

// processList processes and indexes a file listing
func (i *Indexable) processList(ctx context.Context, list *shell.UnixLsObject, references []indexer.Reference) (err error) {
	now := nowISO()

	switch list.Type {
	case "File":
		// Add to file crawl queue with high priority
		fileArgs := &Args{
			Hash:       i.Hash,
			Name:       i.Name,
{'fraction_non_alphanumeric': 0.13366521468574985, 'fraction_numerical': 0.0004978220286247666, 'mean_word_length': 3.388858547241944, 'pattern_counts': {'":': 8, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2600403', 'n_tokens_mistral': 2983, 'n_tokens_neox': 2709, 'n_words': 849}