<commit_before>package crawler

import (
	"net"
	"net/url"
	// "path"
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index panic to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		log.Printf("URL error: %v", uerr)

		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}<commit_msg>Only log temporary errors when not returning them.<commit_after>package crawler

import (
	"net"
	"net/url"
	// "path"
	"context"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs/go-ipfs-api"
	"log"
	"strings"
	"syscall"
	"time"
)

// Indexable consists of args with a Crawler
type Indexable struct {
	*Crawler
	*Args
}

// String returns '<hash>' (<name>)
func (i *Indexable) String() string {
	if i.Name != "" {
		return fmt.Sprintf("'%s' (%s)", i.Hash, i.Name)
	}
	return fmt.Sprintf("'%s' (Unnamed)", i.Hash)
}

// handleShellError handles IPFS shell errors; returns try again bool and original error
func (i *Indexable) handleShellError(ctx context.Context, err error) (bool, error) {
	if _, ok := err.(*shell.Error); ok && (strings.Contains(err.Error(), "proto") ||
		strings.Contains(err.Error(), "unrecognized type") ||
		strings.Contains(err.Error(), "not a valid merkledag node")) {

		// Attempt to index panic to prevent re-indexing
		i.indexInvalid(ctx, err)

		// Don't try again, return error
		return false, err
	}

	// Different error, attempt handling as URL error
	return i.handleURLError(err)
}

// handleURLError handles HTTP errors graceously, returns try again bool and original error
func (i *Indexable) handleURLError(err error) (bool, error) {
	if uerr, ok := err.(*url.Error); ok {
		if uerr.Timeout() {
			// Fail on timeouts
			return false, err
		}

		if uerr.Temporary() {
			// Retry on other temp errors
			log.Printf("Temporary URL error: %v", uerr)
			return true, nil
		}

		// Somehow, the errors below are not temp errors !?
		switch t := uerr.Err.(type) {
		case *net.OpError:
			if t.Op == "dial" {
				log.Printf("Unknown host %v", t)
				return true, nil

			} else if t.Op == "read" {
				log.Printf("Connection refused %v", t)
				return true, nil
			}

		case syscall.Errno:
			if t == syscall.ECONNREFUSED {
				log.Printf("Connection refused %v", t)
				return true, nil
			}
{'fraction_non_alphanumeric': 0.1476178078625358, 'fraction_numerical': 0.0, 'mean_word_length': 3.3958810068649887, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4698082', 'n_tokens_mistral': 1478, 'n_tokens_neox': 1334, 'n_words': 408}