<commit_before>package crawler

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"
)

type metadata map[string]interface{}

// filenameURL returns an IPFS reference including a filename, if available.
// e.g. /ipfs/<parent_hash>/my_file.jpg instead of /ipfs/<file_hash>/
// This helps Tika with file type detection.
func (i *Indexable) getFilenameURL() (path string) {
	if i.Name != "" && i.ParentHash != "" {
		return fmt.Sprintf("/ipfs/%s/%s", i.ParentHash, i.Name)
	}

	// No name & parent hash available
	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

func (i *Indexable) retryingGet(url string) (resp *http.Response, err error) {
	client := http.Client{
		Timeout: i.Config.IpfsTikaTimeout,
	}

	tryAgain := true
	for tryAgain {
		log.Printf("Fetching metadata from '%s'", url)
		resp, err = client.Get(url)<commit_msg>Be less tolerant; a ParentHash implies a Name.<commit_after>package crawler

import (
	"encoding/json"
	"fmt"
	"log"
	"net/http"
	"time"
)

type metadata map[string]interface{}

// filenameURL returns an IPFS reference including a filename, if available.
// e.g. /ipfs/<parent_hash>/my_file.jpg instead of /ipfs/<file_hash>/
// This helps Tika with file type detection.
func (i *Indexable) getFilenameURL() (path string) {
	if i.ParentHash != "" {
		if i.Name == "" {
			panic("ParentHash set but no Name for Indexable.")
		}

		return fmt.Sprintf("/ipfs/%s/%s", i.ParentHash, i.Name)
	}

	return fmt.Sprintf("/ipfs/%s", i.Hash)
}

func (i *Indexable) retryingGet(url string) (resp *http.Response, err error) {
	client := http.Client{
		Timeout: i.Config.IpfsTikaTimeout,
	}

	tryAgain := true
	for tryAgain {
		log.Printf("Fetching metadata from '%s'", url)
		resp, err = client.Get(url)
{'fraction_non_alphanumeric': 0.15384615384615385, 'fraction_numerical': 0.0, 'mean_word_length': 3.9197707736389686, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3470222', 'n_tokens_mistral': 627, 'n_tokens_neox': 609, 'n_words': 167}