<commit_before>package main

import (
	"flag"
	"os"
	"runtime"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/TLS-Observer/certificate"
	"github.com/mozilla/TLS-Observer/config"
	"github.com/mozilla/TLS-Observer/connection"
	"github.com/mozilla/TLS-Observer/logger"
	"github.com/mozilla/TLS-Observer/modules/amqpmodule"
	pg "github.com/mozilla/TLS-Observer/modules/postgresmodule"
	"github.com/mozilla/TLS-Observer/worker"
)

const rxQueue = "cert_rx_queue"
const rxRoutKey = "scan_ready"

var broker *amqpmodule.Broker
var db *pg.DB
var log = logger.GetLogger()

func main() {

	conf := config.ObserverConfig{}

	var cfgFile, cipherscan string
	var debug bool
	flag.StringVar(&cfgFile, "c", "/etc/observer/observer.cfg", "Input file csv format")
	flag.StringVar(&cipherscan, "b", "/etc/observer/cipherscan/cipherscan", "Cipherscan binary location")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()

	if debug {
		logger.SetLevelToDebug()
	}

	_, err := os.Stat(cfgFile)
	log.Fatal("Missing configuration file from '-c' or /etc/observer/observer.cfg")

	_, err = os.Stat(cipherscan)
	if err != nil {
		log.Println("Could not locate cipherscan binary in ", cipherscan, ".")
		log.Println("TLS Connection capabilities are not available.")
	}

	conf, err = config.ObserverConfigLoad(cfgFile)
	if err != nil {
		conf = config.GetObserverDefaults()
	}

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores * conf.General.GoRoutines)

	db, err = pg.RegisterConnection(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")

	log.WithFields(logrus.Fields{
		"error": err.Error(),
	}).Fatal("Failed to connect to database")

	broker, err = amqpmodule.RegisterURL(conf.General.RabbitMQRelay)

	log.WithFields(logrus.Fields{
		"error": err.Error(),
	}).Fatal("Failed to register RabbitMQ")

	msgs, err := broker.Consume(rxQueue, rxRoutKey)

	certificate.Setup(conf, db)

	for d := range msgs {
<commit_msg>replaced a forgotten log.<commit_after>package main

import (
	"flag"
	"os"
	"runtime"
	"strconv"
	"time"

	"github.com/Sirupsen/logrus"

	"github.com/mozilla/TLS-Observer/certificate"
	"github.com/mozilla/TLS-Observer/config"
	"github.com/mozilla/TLS-Observer/connection"
	"github.com/mozilla/TLS-Observer/logger"
	"github.com/mozilla/TLS-Observer/modules/amqpmodule"
	pg "github.com/mozilla/TLS-Observer/modules/postgresmodule"
	"github.com/mozilla/TLS-Observer/worker"
)

const rxQueue = "cert_rx_queue"
const rxRoutKey = "scan_ready"

var broker *amqpmodule.Broker
var db *pg.DB
var log = logger.GetLogger()

func main() {

	conf := config.ObserverConfig{}

	var cfgFile, cipherscan string
	var debug bool
	flag.StringVar(&cfgFile, "c", "/etc/observer/observer.cfg", "Input file csv format")
	flag.StringVar(&cipherscan, "b", "/etc/observer/cipherscan/cipherscan", "Cipherscan binary location")
	flag.BoolVar(&debug, "debug", false, "Set debug logging")
	flag.Parse()

	if debug {
		logger.SetLevelToDebug()
	}

	_, err := os.Stat(cfgFile)
	log.Fatal("Missing configuration file from '-c' or /etc/observer/observer.cfg")

	_, err = os.Stat(cipherscan)
	if err != nil {
		log.WithFields(logrus.Fields{
			"error": err.Error(),
		}).Error("Could not locate cipherscan executable. TLS connection capabilities will not be available.")
	}

	conf, err = config.ObserverConfigLoad(cfgFile)
	if err != nil {
		conf = config.GetObserverDefaults()
	}

	cores := runtime.NumCPU()
	runtime.GOMAXPROCS(cores * conf.General.GoRoutines)

	db, err = pg.RegisterConnection(conf.General.PostgresDB, conf.General.PostgresUser, conf.General.PostgresPass, conf.General.Postgres, "disable")

	log.WithFields(logrus.Fields{
		"error": err.Error(),
	}).Fatal("Failed to connect to database")

	broker, err = amqpmodule.RegisterURL(conf.General.RabbitMQRelay)

	log.WithFields(logrus.Fields{
		"error": err.Error(),
	}).Fatal("Failed to register RabbitMQ")

	msgs, err := broker.Consume(rxQueue, rxRoutKey)

	certificate.Setup(conf, db)

	for d := range msgs {

{'fraction_non_alphanumeric': 0.14469453376205788, 'fraction_numerical': 0.0, 'mean_word_length': 4.860869565217391, 'pattern_counts': {'":': 5, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2704699', 'n_tokens_mistral': 1499, 'n_tokens_neox': 1371, 'n_words': 285}