<commit_before>/**Copyright Blue Medora Inc. 2016**/

package main

import (
	"github.com/BlueMedora/bluemedora-firehose-nozzle/bluemedorafirehosenozzle"
	"github.com/BlueMedora/bluemedora-firehose-nozzle/logger"
	"github.com/BlueMedora/bluemedora-firehose-nozzle/nozzleconfiguration"
	"github.com/BlueMedora/bluemedora-firehose-nozzle/webserver"
)

const (
	defaultConfigLocation = "./config/bluemedora-firehose-nozzle.json"

	defaultLogDirectory = "./logs"
	nozzleLogFile       = "bm_nozzle.log"
	nozzleLogName       = "bm_firehose_nozzle"

	webserverLogFile = "bm_server.log"
	webserverLogName = "bm_server"
)

func main() {
    logger.CreateLogDirectory(defaultLogDirectory)
    
	logger := logger.New(defaultLogDirectory, nozzleLogFile, nozzleLogName)
	logger.Debug("working log")

	//Read in config
	config, err := nozzleconfiguration.New(defaultConfigLocation, logger)
	if err != nil {
		logger.Fatalf("Error parsing config file: %s", err.Error())
	}

    //Setup and start nozzle
	server := createWebServer(config)

	nozzle := bluemedorafirehosenozzle.New(config, server, logger)
	nozzle.Start()

	if err != nil {
		logger.Fatalf("Error while running nozzle: %s", err.Error())
	}
}

func createWebServer(config *nozzleconfiguration.NozzleConfiguration) *webserver.WebServer {
	logger := logger.New(defaultLogDirectory, webserverLogFile, webserverLogName)
	return webserver.New(config, logger)
}<commit_msg>Added mode flag to main to allow webserver to be stood up alone for debugging<commit_after>/**Copyright Blue Medora Inc. 2016**/

package main

import (
	"flag"
	
	"github.com/BlueMedora/bluemedora-firehose-nozzle/bluemedorafirehosenozzle"
	"github.com/BlueMedora/bluemedora-firehose-nozzle/logger"
	"github.com/BlueMedora/bluemedora-firehose-nozzle/nozzleconfiguration"
	"github.com/BlueMedora/bluemedora-firehose-nozzle/webserver"
)

const (
	defaultConfigLocation = "./config/bluemedora-firehose-nozzle.json"

	defaultLogDirectory = "./logs"
	nozzleLogFile       = "bm_nozzle.log"
	nozzleLogName       = "bm_firehose_nozzle"

	webserverLogFile = "bm_server.log"
	webserverLogName = "bm_server"
)

var (
	//Mode to run nozzle in. Webserver mode is for debugging purposes only
	runMode = flag.String("mode", "normal", "Mode to run nozzle `normal` or `webserver`")
)

func main() {
	flag.Parse()
	
	if *runMode == "normal" {
		normalSetup()
	} else if *runMode == "webserver" {
		standUpWebServer()
	}
}

func normalSetup() {
	logger.CreateLogDirectory(defaultLogDirectory)
    
	logger := logger.New(defaultLogDirectory, nozzleLogFile, nozzleLogName)
	logger.Debug("working log")

	//Read in config
	config, err := nozzleconfiguration.New(defaultConfigLocation, logger)
	if err != nil {
		logger.Fatalf("Error parsing config file: %s", err.Error())
	}

    //Setup and start nozzle
	server := createWebServer(config)

	nozzle := bluemedorafirehosenozzle.New(config, server, logger)
	nozzle.Start()

	if err != nil {
		logger.Fatalf("Error while running nozzle: %s", err.Error())
	}
}

func standUpWebServer() {
	logger := logger.New(defaultLogDirectory, webserverLogFile, webserverLogName)
    
    //Read in config
	config, err := nozzleconfiguration.New(defaultConfigLocation, logger)
	if err != nil {
		logger.Fatalf("Error parsing config file: %s", err.Error())
	}
    
    server := webserver.New(config, logger)
    
    logger.Info("Starting webserver")
    errors := server.Start(webserver.DefaultKeyLocation, webserver.DefaultCertLocation)
    
    select {
        case err := <-errors:
            logger.Fatalf("Error while running server: %s", err.Error())
    }
}

func createWebServer(config *nozzleconfiguration.NozzleConfiguration) *webserver.WebServer {
	logger := logger.New(defaultLogDirectory, webserverLogFile, webserverLogName)
	return webserver.New(config, logger)
}

{'fraction_non_alphanumeric': 0.11928533893851813, 'fraction_numerical': 0.0021019442984760903, 'mean_word_length': 4.724812030075188, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6040222', 'n_tokens_mistral': 1344, 'n_tokens_neox': 1254, 'n_words': 252}