<commit_before>package config

import (
	"fmt"
	"github.com/s-rah/onionscan/crawldb"
	"log"
	"os"
	"path/filepath"
	"time"
)

type OnionScanConfig struct {
	TorProxyAddress string
	Depth           int
	Fingerprint     bool
	Timeout         time.Duration
	Verbose         bool
	Database        *crawldb.CrawlDB
	RescanDuration  time.Duration
	Scans           []string
	CrawlConfigs    map[string]CrawlConfig
}

func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int, database string, scans []string, crawlconfigdir string, verbose bool) *OnionScanConfig {
	osc := new(OnionScanConfig)
	osc.TorProxyAddress = torProxyAddress
	osc.Depth = directoryDepth
	osc.Fingerprint = fingerprint
	osc.Timeout = time.Duration(time.Second * time.Duration(timeout))
	osc.Verbose = verbose
	osc.Database = new(crawldb.CrawlDB)
	osc.Database.NewDB(database)
	osc.RescanDuration = time.Hour * -100
	osc.Scans = scans
	osc.CrawlConfigs = make(map[string]CrawlConfig)

	visit := func(path string, f os.FileInfo, err error) error {
		if !f.IsDir() {
			cc, err := LoadCrawlConfig(path)
			if err == nil {
				osc.LogInfo(fmt.Sprintf("Loading Crawl Config for %s %v", cc.Onion, cc))
				osc.CrawlConfigs[cc.Onion] = cc
			}
		}
		return nil
	}

	filepath.Walk(crawlconfigdir, visit)

	return osc
}

func (os *OnionScanConfig) LogInfo(message string) {
	if os.Verbose {
		log.Printf("INFO: %v", message)
	}
}

func (os *OnionScanConfig) LogError(err error) {
	log.Printf("ERROR: %v", err)
}<commit_msg>Don't Load Crawl Config when it isn't set<commit_after>package config

import (
	"fmt"
	"github.com/s-rah/onionscan/crawldb"
	"log"
	"os"
	"path/filepath"
	"time"
)

type OnionScanConfig struct {
	TorProxyAddress string
	Depth           int
	Fingerprint     bool
	Timeout         time.Duration
	Verbose         bool
	Database        *crawldb.CrawlDB
	RescanDuration  time.Duration
	Scans           []string
	CrawlConfigs    map[string]CrawlConfig
}

func Configure(torProxyAddress string, directoryDepth int, fingerprint bool, timeout int, database string, scans []string, crawlconfigdir string, verbose bool) *OnionScanConfig {
	osc := new(OnionScanConfig)
	osc.TorProxyAddress = torProxyAddress
	osc.Depth = directoryDepth
	osc.Fingerprint = fingerprint
	osc.Timeout = time.Duration(time.Second * time.Duration(timeout))
	osc.Verbose = verbose
	osc.Database = new(crawldb.CrawlDB)
	osc.Database.NewDB(database)
	osc.RescanDuration = time.Hour * -100
	osc.Scans = scans
	osc.CrawlConfigs = make(map[string]CrawlConfig)

	visit := func(path string, f os.FileInfo, err error) error {
		if !f.IsDir() {
			cc, err := LoadCrawlConfig(path)
			if err == nil {
				osc.LogInfo(fmt.Sprintf("Loading Crawl Config for %s %v", cc.Onion, cc))
				osc.CrawlConfigs[cc.Onion] = cc
			}
		}
		return nil
	}

	if crawlconfigdir != "" {
		filepath.Walk(crawlconfigdir, visit)
	}

	return osc
}

func (os *OnionScanConfig) LogInfo(message string) {
	if os.Verbose {
		log.Printf("INFO: %v", message)
	}
}

func (os *OnionScanConfig) LogError(err error) {
	log.Printf("ERROR: %v", err)
}
{'fraction_non_alphanumeric': 0.11295034079844206, 'fraction_numerical': 0.0019474196689386564, 'mean_word_length': 3.868878357030016, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7217595', 'n_tokens_mistral': 1171, 'n_tokens_neox': 1097, 'n_words': 251}