<commit_before>package app

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"
	"github.com/olebedev/config"
	"github.com/spf13/cobra"

	"github.com/theduke/go-apperror"
	db "github.com/theduke/go-dukedb"

	kit "github.com/theduke/go-appkit"
	//"github.com/theduke/go-appkit/caches"
	"github.com/theduke/go-appkit/crawler"
	"github.com/theduke/go-appkit/resources"

	"github.com/theduke/go-appkit/frontends/jsonapi"

	"github.com/theduke/go-appkit/email"
	"github.com/theduke/go-appkit/email/gomail"
	emaillog "github.com/theduke/go-appkit/email/log"
)

type App struct {
	env   string
	debug bool

	logger *logrus.Logger

	deps      kit.Dependencies
	frontends map[string]kit.Frontend

	methods map[string]kit.Method

	beforeMiddlewares []kit.RequestHandler
	afterMiddlewares  []kit.AfterRequestMiddleware

	serverErrorHandler kit.AfterRequestMiddleware
	notFoundHandler    kit.RequestHandler

	sessionManager *SessionManager

	router *httprouter.Router

	Cli *cobra.Command

	shutDownChannel chan bool
}

// Ensure App implements App interface.
var _ kit.App = (*App)(nil)

func NewApp(cfgPaths ...string) *App {
	app := App{
		deps:              NewDependencies(),
		frontends:         make(map[string]kit.Frontend),
		methods:           make(map[string]kit.Method),
		beforeMiddlewares: make([]kit.RequestHandler, 0),
		afterMiddlewares:  make([]kit.AfterRequestMiddleware, 0),

		router: httprouter.New(),

		shutDownChannel: make(chan bool),
	}

	// Configure logger.
	app.SetLogger(&logrus.Logger{
		Out:       os.Stderr,
		Formatter: new(logrus.TextFormatter),
		Hooks:     make(logrus.LevelHooks),
		Level:     logrus.DebugLevel,
	})

	app.InitCli()

	configPath := "config.yaml"
	if len(cfgPaths) > 0 {
		configPath = cfgPaths[0]
	}
	app.ReadConfig(configPath)

	app.Defaults()

	return &app
}

func (a *App) Defaults() {
	// EmailService setup.
	a.buildEmailService()

	a.RegisterMethod(createMethod())
	a.RegisterMethod(updateMethod())
	a.RegisterMethod(deleteMethod())
	a.RegisterMethod(queryMethod())

	a.RegisterBeforeMiddleware(RequestTraceMiddleware)
	a.RegisterBeforeMiddleware(AuthenticationMiddleware)

	a.RegisterAfterMiddleware(ServerErrorMiddleware)
	a.RegisterAfterMiddleware(RequestTraceAfterMiddleware)
	a.RegisterAfterMiddleware(RequestLoggerMiddleware)

	a.RegisterFrontend(jsonapi.New(a))
}

func (a *App) ENV() string {
	return a.env
}

func (a *App) SetENV(x string) {
	a.env = x
}

func (a *App) Debug() bool {
	return a.debug
}

func (a *App) SetDebug(x bool) {
	a.debug = x
}

func (a *App) Dependencies() kit.Dependencies {
	return a.deps
}

func (a *App) Logger() *logrus.Logger {
	return a.logger
}

func (a *App) SetLogger(x *logrus.Logger) {
	a.logger = x
	a.deps.SetLogger(x)
}

func (a *App) Router() *httprouter.Router {
	return a.router
}

func (a *App) TmpDir() string {
	return a.deps.Config().UString("tmpDir", "tmp")
}

/**
 * Config.
 */

func (a *App) Config() *config.Config {
	return a.deps.Config()
}

func (a *App) SetConfig(x *config.Config) {
	a.deps.SetConfig(x)
}

func (a *App) ReadConfig(path string) {
	if path == "" {
		path = "config.yaml"
	}

	var cfg *config.Config

	if f, err := os.Open(path); err != nil {
		a.Logger().Infof("Could not find or read config at '%v' - Using default settings\n", path)
	} else {
		defer f.Close()
		content, err := ioutil.ReadAll(f)
		if err != nil {
			a.Logger().Panicf("Could not read config at '%v': %v\n", path, err)
		} else {
			cfg, err = config.ParseYaml(string(content))
			if err != nil {
				a.Logger().Panicf("YAML error while parsing config at '%v': %v\n", path, err)
			}
		}
	}

	if cfg == nil {
		defaultConfig := "ENV: dev\ntmpDir: tmp"

		c, _ := config.ParseYaml(defaultConfig)
		cfg = c
	}

	cfg.Env()

	// Set default values if not present.
	env := cfg.UString("ENV", "dev")
	if env == "dev" {
		a.Logger().Info("No environment specified, defaulting to 'dev'")
		a.debug = true
	}
	a.env = env

	// Fill in default values into the config and ensure they are valid.

	// Ensure a tmp directory exists and is readable.
	tmpDir := cfg.UString("tmpDir", "tmp")
	if err := os.MkdirAll(tmpDir, 0777); err != nil {
		a.Logger().Panicf("Could not read or create tmp dir at '%v': %v", tmpDir, err)
	}

	a.deps.SetConfig(cfg)
}

func (a *App) PrepareBackends() {
	backends := a.deps.Backends()
	for name := range backends {
		backends[name].BuildRelationshipInfo()
	}
}

func (a *App) PrepareForRun() {
	a.PrepareBackends()

	// Auto migrate if enabled or not explicitly disabled and env is debug.
	if auto, err := a.Config().Bool("autoRunMigrations"); (err == nil && auto) || (err != nil && a.env == "dev") {
		if err := a.MigrateAllBackends(false); err != nil {
			a.Logger().Errorf("Migration FAILED: %v\n", err)
		}
	}

	// Initialize frontends.
	for name := range a.frontends {
		if err := a.frontends[name].Init(); err != nil {
			a.Logger().Panicf("Error while initializing frontend %v: %v", name, err)
		}
	}
}

func (a *App) Run() {
	a.PrepareForRun()

	if a.notFoundHandler == nil {
		a.notFoundHandler = notFoundHandler
	}

	// Install not found handler.
	a.router.NotFound = &HttpHandlerStruct{
		App:     a,
		Handler: a.notFoundHandler,
	}

	// Install handler for index.
	indexTpl, err := getIndexTpl(a)
	if err != nil {
		a.Logger().Panic(err)
	}

	a.router.GET("/", func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		httpHandler(w, r, params, a, func(kit.App, kit.Request) (kit.Response, bool) {
			return &kit.AppResponse{
				RawData: indexTpl,
			}, false
		})
	})

	// Serve files routes.
	serveFiles := a.Config().UMap("serveFiles")
	for route := range serveFiles {
		path, ok := serveFiles[route].(string)
		if !ok {
			a.Logger().Error("Config error: serveFiles configuration invalid: Must be map/dictionary with paths")
			continue
		}
		a.ServeFiles(route, path)
	}

	// Register route for method calls.
	methodHandler := func(a kit.App, r kit.Request) (kit.Response, bool) {
		r.ParseJsonData()

		var response kit.Response

		responder := func(r kit.Response) {
			response = r
		}

		method := r.GetContext().String("name")

		finishedChannel, err := a.RunMethod(method, r, responder, true)
		if err != nil {
			return &kit.AppResponse{
				Error: err,
			}, false
		}
		<-finishedChannel

		return response, false
	}

	a.router.POST("/api/method/:name", func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		httpHandler(w, r, params, a, methodHandler)
	})

	// Run the session manager.
	a.sessionManager = NewSessionManager(a)
	a.sessionManager.Run()

	// Run frontends.
	for name := range a.frontends {
		if err := a.frontends[name].Start(); err != nil {
			a.Logger().Panicf("Could not start frontend %v: %v", name, err)
		}
	}

	url := a.Config().UString("host", "localhost") + ":" + a.Config().UString("port", "8000")
	a.Logger().Infof("Serving on %v", url)

	go func() {
		err2 := http.ListenAndServe(url, a.router)
		if err2 != nil {
			a.Logger().Panicf("Could not start server: %v\n", err)
		}
	}()

	// Crawl on startup if enabled.
	if a.Config().UBool("crawler.onRun", false) {
		a.RunCrawler()
	}

	<-a.shutDownChannel
}

/**
 * Crawling.
 */

func (a *App) RunCrawler() {
	recrawlInterval := a.Config().UInt("crawler.recrawlInterval", 0)

	a.Logger().Infof("Running crawler with recrawl interval %v", recrawlInterval)

	go func() {
		for {
			a.Crawl()
			if recrawlInterval == 0 {
				break
			} else {
				time.Sleep(time.Second * time.Duration(recrawlInterval))
				a.Crawl()
			}
		}
	}()
}

func (a *App) Crawl() {
	concurrentRequests := a.Config().UInt("crawler.concurrentRequests", 5)
	host := a.Config().UString("url")
	if host == "" {
		a.Logger().Error("Can't crawl because 'url' setting is not specified")
		return
	}

	hosts := []string{host}
	startUrls := []string{"http://" + host + "/"}
	crawler := crawler.New(concurrentRequests, hosts, startUrls)
	crawler.Logger = a.logger

	crawler.Run()
}

/**
 * Serve files.
 */

func (a App) ServeFiles(route string, path string) {
	a.Logger().Debugf("Serving files from directory '%v' at route '%v'", path, route)

	server := http.FileServer(http.Dir(path))
	a.Router().GET(route+"/*path", func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		// Fix the url.
		r.URL.Path = params.ByName("path")
		server.ServeHTTP(w, r)
	})
}

/**
 * Backends.
 */

func (a *App) RegisterBackend(b db.Backend) {
	if b.GetLogger() == nil {
		b.SetLogger(a.Logger())
	}
	a.deps.AddBackend(b)<commit_msg>Now creating default fileservice, cache with fs backend and userservice<commit_after>package app

import (
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"time"

	"github.com/Sirupsen/logrus"
	"github.com/julienschmidt/httprouter"
	"github.com/olebedev/config"
	"github.com/spf13/cobra"

	"github.com/theduke/go-apperror"
	db "github.com/theduke/go-dukedb"

	kit "github.com/theduke/go-appkit"
	"github.com/theduke/go-appkit/caches/fs"
	"github.com/theduke/go-appkit/crawler"
	"github.com/theduke/go-appkit/files"
	"github.com/theduke/go-appkit/resources"
	"github.com/theduke/go-appkit/users"

	"github.com/theduke/go-appkit/frontends/jsonapi"

	"github.com/theduke/go-appkit/email"
	"github.com/theduke/go-appkit/email/gomail"
	emaillog "github.com/theduke/go-appkit/email/log"
)

type App struct {
	env   string
	debug bool

	logger *logrus.Logger

	deps      kit.Dependencies
	frontends map[string]kit.Frontend

	methods map[string]kit.Method

	beforeMiddlewares []kit.RequestHandler
	afterMiddlewares  []kit.AfterRequestMiddleware

	serverErrorHandler kit.AfterRequestMiddleware
	notFoundHandler    kit.RequestHandler

	sessionManager *SessionManager

	router *httprouter.Router

	Cli *cobra.Command

	shutDownChannel chan bool
}

// Ensure App implements App interface.
var _ kit.App = (*App)(nil)

func NewApp(cfgPaths ...string) *App {
	app := App{
		deps:              NewDependencies(),
		frontends:         make(map[string]kit.Frontend),
		methods:           make(map[string]kit.Method),
		beforeMiddlewares: make([]kit.RequestHandler, 0),
		afterMiddlewares:  make([]kit.AfterRequestMiddleware, 0),

		router: httprouter.New(),

		shutDownChannel: make(chan bool),
	}

	// Configure logger.
	app.SetLogger(&logrus.Logger{
		Out:       os.Stderr,
		Formatter: new(logrus.TextFormatter),
		Hooks:     make(logrus.LevelHooks),
		Level:     logrus.DebugLevel,
	})

	app.InitCli()

	configPath := "config.yaml"
	if len(cfgPaths) > 0 {
		configPath = cfgPaths[0]
	}
	app.ReadConfig(configPath)

	app.Defaults()

	return &app
}

func (a *App) Defaults() {
	// EmailService setup.
	a.buildEmailService()

	a.RegisterMethod(createMethod())
	a.RegisterMethod(updateMethod())
	a.RegisterMethod(deleteMethod())
	a.RegisterMethod(queryMethod())

	a.RegisterBeforeMiddleware(RequestTraceMiddleware)
	a.RegisterBeforeMiddleware(AuthenticationMiddleware)

	a.RegisterAfterMiddleware(ServerErrorMiddleware)
	a.RegisterAfterMiddleware(RequestTraceAfterMiddleware)
	a.RegisterAfterMiddleware(RequestLoggerMiddleware)

	// Register file service with fs backend.

	dir := a.Config().UString("files.dir")
	if dir == "" {
		dir = a.Config().UString("dataDir", "data") + "/" + "files"
	}
	fileService := files.NewFileServiceWithFs(nil, dir)
	a.RegisterFileService(fileService)

	// Register fs cache.

	// Build cache.
	dir = a.Config().UString("caches.fs.dir")
	if dir == "" {
		dir = a.TmpDir() + "/" + "cache"
	}
	fsCache, err := fs.New(dir)
	if err != nil {
		panic("Could not initialize fs cache: " + err.Error())
	}
	a.RegisterCache(fsCache)

	a.RegisterFrontend(jsonapi.New(a))
}

func (a *App) defaultUserService(b db.Backend) {
	s := users.NewService(nil, b, nil)
	a.RegisterUserService(s)
}

func (a *App) ENV() string {
	return a.env
}

func (a *App) SetENV(x string) {
	a.env = x
}

func (a *App) Debug() bool {
	return a.debug
}

func (a *App) SetDebug(x bool) {
	a.debug = x
}

func (a *App) Dependencies() kit.Dependencies {
	return a.deps
}

func (a *App) Logger() *logrus.Logger {
	return a.logger
}

func (a *App) SetLogger(x *logrus.Logger) {
	a.logger = x
	a.deps.SetLogger(x)
}

func (a *App) Router() *httprouter.Router {
	return a.router
}

func (a *App) TmpDir() string {
	return a.deps.Config().UString("tmpDir", "tmp")
}

/**
 * Config.
 */

func (a *App) Config() *config.Config {
	return a.deps.Config()
}

func (a *App) SetConfig(x *config.Config) {
	a.deps.SetConfig(x)
}

func (a *App) ReadConfig(path string) {
	if path == "" {
		path = "config.yaml"
	}

	var cfg *config.Config

	if f, err := os.Open(path); err != nil {
		a.Logger().Infof("Could not find or read config at '%v' - Using default settings\n", path)
	} else {
		defer f.Close()
		content, err := ioutil.ReadAll(f)
		if err != nil {
			a.Logger().Panicf("Could not read config at '%v': %v\n", path, err)
		} else {
			cfg, err = config.ParseYaml(string(content))
			if err != nil {
				a.Logger().Panicf("YAML error while parsing config at '%v': %v\n", path, err)
			}
		}
	}

	if cfg == nil {
		defaultConfig := "ENV: dev\ntmpDir: tmp"

		c, _ := config.ParseYaml(defaultConfig)
		cfg = c
	}

	cfg.Env()

	// Set default values if not present.
	env := cfg.UString("ENV", "dev")
	if env == "dev" {
		a.Logger().Info("No environment specified, defaulting to 'dev'")
		a.debug = true
	}
	a.env = env

	// Fill in default values into the config and ensure they are valid.

	// Ensure a tmp directory exists and is readable.
	tmpDir := cfg.UString("tmpDir", "tmp")
	if err := os.MkdirAll(tmpDir, 0777); err != nil {
		a.Logger().Panicf("Could not read or create tmp dir at '%v': %v", tmpDir, err)
	}

	a.deps.SetConfig(cfg)
}

func (a *App) PrepareBackends() {
	backends := a.deps.Backends()
	for name := range backends {
		backends[name].BuildRelationshipInfo()
	}
}

func (a *App) PrepareForRun() {
	a.PrepareBackends()

	// Auto migrate if enabled or not explicitly disabled and env is debug.
	if auto, err := a.Config().Bool("autoRunMigrations"); (err == nil && auto) || (err != nil && a.env == "dev") {
		if err := a.MigrateAllBackends(false); err != nil {
			a.Logger().Errorf("Migration FAILED: %v\n", err)
		}
	}

	// Initialize frontends.
	for name := range a.frontends {
		if err := a.frontends[name].Init(); err != nil {
			a.Logger().Panicf("Error while initializing frontend %v: %v", name, err)
		}
	}
}

func (a *App) Run() {
	a.PrepareForRun()

	if a.notFoundHandler == nil {
		a.notFoundHandler = notFoundHandler
	}

	// Install not found handler.
	a.router.NotFound = &HttpHandlerStruct{
		App:     a,
		Handler: a.notFoundHandler,
	}

	// Install handler for index.
	indexTpl, err := getIndexTpl(a)
	if err != nil {
		a.Logger().Panic(err)
	}

	a.router.GET("/", func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		httpHandler(w, r, params, a, func(kit.App, kit.Request) (kit.Response, bool) {
			return &kit.AppResponse{
				RawData: indexTpl,
			}, false
		})
	})

	// Serve files routes.
	serveFiles := a.Config().UMap("serveFiles")
	for route := range serveFiles {
		path, ok := serveFiles[route].(string)
		if !ok {
			a.Logger().Error("Config error: serveFiles configuration invalid: Must be map/dictionary with paths")
			continue
		}
		a.ServeFiles(route, path)
	}

	// Register route for method calls.
	methodHandler := func(a kit.App, r kit.Request) (kit.Response, bool) {
		r.ParseJsonData()

		var response kit.Response

		responder := func(r kit.Response) {
			response = r
		}

		method := r.GetContext().String("name")

		finishedChannel, err := a.RunMethod(method, r, responder, true)
		if err != nil {
			return &kit.AppResponse{
				Error: err,
			}, false
		}
		<-finishedChannel

		return response, false
	}

	a.router.POST("/api/method/:name", func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		httpHandler(w, r, params, a, methodHandler)
	})

	// Run the session manager.
	a.sessionManager = NewSessionManager(a)
	a.sessionManager.Run()

	// Run frontends.
	for name := range a.frontends {
		if err := a.frontends[name].Start(); err != nil {
			a.Logger().Panicf("Could not start frontend %v: %v", name, err)
		}
	}

	url := a.Config().UString("host", "localhost") + ":" + a.Config().UString("port", "8000")
	a.Logger().Infof("Serving on %v", url)

	go func() {
		err2 := http.ListenAndServe(url, a.router)
		if err2 != nil {
			a.Logger().Panicf("Could not start server: %v\n", err)
		}
	}()

	// Crawl on startup if enabled.
	if a.Config().UBool("crawler.onRun", false) {
		a.RunCrawler()
	}

	<-a.shutDownChannel
}

/**
 * Crawling.
 */

func (a *App) RunCrawler() {
	recrawlInterval := a.Config().UInt("crawler.recrawlInterval", 0)

	a.Logger().Infof("Running crawler with recrawl interval %v", recrawlInterval)

	go func() {
		for {
			a.Crawl()
			if recrawlInterval == 0 {
				break
			} else {
				time.Sleep(time.Second * time.Duration(recrawlInterval))
				a.Crawl()
			}
		}
	}()
}

func (a *App) Crawl() {
	concurrentRequests := a.Config().UInt("crawler.concurrentRequests", 5)
	host := a.Config().UString("url")
	if host == "" {
		a.Logger().Error("Can't crawl because 'url' setting is not specified")
		return
	}

	hosts := []string{host}
	startUrls := []string{"http://" + host + "/"}
	crawler := crawler.New(concurrentRequests, hosts, startUrls)
	crawler.Logger = a.logger

	crawler.Run()
}

/**
 * Serve files.
 */

func (a App) ServeFiles(route string, path string) {
	a.Logger().Debugf("Serving files from directory '%v' at route '%v'", path, route)

	server := http.FileServer(http.Dir(path))
	a.Router().GET(route+"/*path", func(w http.ResponseWriter, r *http.Request, params httprouter.Params) {
		// Fix the url.
		r.URL.Path = params.ByName("path")
		server.ServeHTTP(w, r)
	})
}

/**
 * Backends.
 */

func (a *App) RegisterBackend(b db.Backend) {
	if b.GetLogger() == nil {
		b.SetLogger(a.Logger())
	}

	// If no backend is registered yet, create a default UserService.
	if a.DefaultBackend() == nil {
		a.defaultUserService(b)
	}

	a.deps.AddBackend(b)
{'fraction_non_alphanumeric': 0.1604629165971179, 'fraction_numerical': 0.0021142825349134814, 'mean_word_length': 3.6954022988505746, 'pattern_counts': {'":': 2, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3766204', 'n_tokens_mistral': 7016, 'n_tokens_neox': 6607, 'n_words': 1497}