<commit_before>package main

import (
	"./common"
	_ "./fetcher"
	"./model"
	"flag"
	"github.com/astaxie/beego/orm"
	_ "github.com/go-sql-driver/mysql"
	"github.com/op/go-logging"
)

var log *logging.Logger

func init() {
	orm.RegisterDriver("mysql", orm.DR_MySQL)
}

func initDatabase(config *common.Config) {
	orm.RegisterDataBase("default", "mysql", config.ConnectString())
}

func invokeFeedFetcher() bool {
	feeds, err := model.RdfFeeds()
	if err != nil {
		log.Error("err: %v\n", err)
		return false
	}

	for _, feed := range feeds {
		log.Info("%s\n", feed.Url)
	}

	return true
}

func main() {
	log = common.GetLogger()

	path := flag.String("c", "./config.toml", "path to config.tml")
	flag.Parse()

	var config, err = common.LoadConfig(*path)
	if err != nil {
		panic(err)
	}

	initDatabase(config)

	// invoke fetcher
	invokeFeedFetcher()
}
<commit_msg>remove panic<commit_after>package main

import (
	"./common"
	_ "./fetcher"
	"./model"
	"flag"
	"github.com/astaxie/beego/orm"
	_ "github.com/go-sql-driver/mysql"
	"github.com/op/go-logging"
	"os"
)

var log *logging.Logger

func init() {
	orm.RegisterDriver("mysql", orm.DR_MySQL)
}

func initDatabase(config *common.Config) {
	orm.RegisterDataBase("default", "mysql", config.ConnectString())
}

func invokeFeedFetcher() bool {
	feeds, err := model.RdfFeeds()
	if err != nil {
		log.Error("err: %v\n", err)
		return false
	}

	for _, feed := range feeds {
		log.Info("%s\n", feed.Url)
	}

	return true
}

func main() {
	log = common.GetLogger()

	path := flag.String("c", "./config.toml", "path to config.tml")
	flag.Parse()

	var config, err = common.LoadConfig(*path)
	if err != nil {
		os.Exit(1)
	}

	initDatabase(config)

	// invoke fetcher
	invokeFeedFetcher()
}

{'fraction_non_alphanumeric': 0.17694524495677233, 'fraction_numerical': 0.0005763688760806917, 'mean_word_length': 3.756164383561644, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5570012', 'n_tokens_mistral': 697, 'n_tokens_neox': 678, 'n_words': 138}