<commit_before>package crawler

import (
	"encoding/json"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs-search/ipfs-search/queue"
	"github.com/ipfs/go-ipfs-api"
)

// Args describe a resource to be crawled
type Args struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

// Crawler consumes file and hash queues and indexes them
type Crawler struct {
	Config *Config

	Shell     *shell.Shell
	Indexer   *indexer.Indexer
	FileQueue *queue.Queue
	HashQueue *queue.Queue
}

// IndexableFromJSON returns and Indexable associated with this crawler based on a JSON blob
func (c *Crawler) IndexableFromJSON(input []byte) (*Indexable, error) {
	// Unmarshall message into crawler Args
	args := &Args{}
	err := json.Unmarshal(input, args)
	if err != nil {
		return nil, err
	}

	return &Indexable{
		Args:    args,
		Crawler: c,
	}, nil

}<commit_msg>Try and catch 404 toward Tika early.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"github.com/ipfs-search/ipfs-search/indexer"
	"github.com/ipfs-search/ipfs-search/queue"
	"github.com/ipfs/go-ipfs-api"
)

// Args describe a resource to be crawled
type Args struct {
	Hash       string
	Name       string
	Size       uint64
	ParentHash string
	ParentName string // This is legacy, should be removed
}

// Crawler consumes file and hash queues and indexes them
type Crawler struct {
	Config *Config

	Shell     *shell.Shell
	Indexer   *indexer.Indexer
	FileQueue *queue.Queue
	HashQueue *queue.Queue
}

// IndexableFromJSON returns and Indexable associated with this crawler based on a JSON blob
func (c *Crawler) IndexableFromJSON(input []byte) (*Indexable, error) {
	// Unmarshall message into crawler Args
	args := &Args{}
	err := json.Unmarshal(input, args)
	if err != nil {
		return nil, err
	}

	// Later down, we assume this hash is set and we're seeing errors where
	// this aparently seems not the case.
	if args.Hash == "" {
		return nil, fmt.Errorf("Empty hash in JSON: %s", input)
	}

	return &Indexable{
		Args:    args,
		Crawler: c,
	}, nil

}
{'fraction_non_alphanumeric': 0.09887005649717515, 'fraction_numerical': 0.003295668549905838, 'mean_word_length': 3.4363256784968685, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5729223', 'n_tokens_mistral': 727, 'n_tokens_neox': 707, 'n_words': 232}