<commit_before>package core2

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/jhunt/go-log"

	"github.com/starkandwayne/shield/core/bus"
	"github.com/starkandwayne/shield/core/fabric"
	"github.com/starkandwayne/shield/core/scheduler"
	"github.com/starkandwayne/shield/core/vault"
	"github.com/starkandwayne/shield/db"
	"github.com/starkandwayne/shield/timespec"
)

func (c Core) Main() {
	/* we need a usable database first */
	c.ConnectToDatabase()
	c.ConfigureMessageBus()

	/* startup cleanup tasks */
	c.CreateFailsafeUser()
	c.ExpireInteractiveSessions()
	c.PrecreateTenants()
	c.CleanupLeftoverTasks()

	/* prepare for operation */
	c.ConnectToVault()
	c.Bind()
	c.StartScheduler()

	log.Infof("INITIALIZATION COMPLETE; entering main loop.")

	fast := time.NewTicker(time.Second * time.Duration(c.Config.Scheduler.FastLoop))
	slow := time.NewTicker(time.Second * time.Duration(c.Config.Scheduler.SlowLoop))
	for {
		select {
		case <-fast.C:
			c.ScheduleBackupTasks()
			c.ScheduleAgentStatusCheckTasks(&db.AgentFilter{Status: "pending"})
			c.RunScheduler()

		case <-slow.C:
			c.CheckArchiveExpiries()
			c.SchedulePurgeTasks()
			c.MarkIrrelevantTasks()
			c.ScheduleAgentStatusCheckTasks(nil)
			c.UpdateGlobalStorageUsage()
			c.UpdateTenantStorageUsage()

			if c.Unlocked() {
				c.PurgeExpiredAPISessions()
				c.ScheduleStorageTestTasks()
			}
		}
	}
}

func (c *Core) ConnectToDatabase() {
	log.Infof("INITIALIZING: connecting to the database...")
	if c.db != nil {
		log.Alertf("ANOMALY: tried to connect to database, but we're already connected...")
		return
	}

	log.Debugf("connecting to database at %s...", c.DataFile("shield.db"))
	db, err := db.Connect(c.DataFile("shield.db"))
	c.MaybeTerminate(err)
	c.db = db

	log.Infof("INITIALIZING: checking database schema...")
	c.MaybeTerminate(c.db.CheckCurrentSchema())

	log.Debugf("connected successfully to database!")
}

func (c *Core) CreateFailsafeUser() {
	log.Infof("INITIALIZING: creating the SHIELD (local auth) failsafe user...")

	if c.Config.API.Failsafe.Username == "" {
		log.Infof("no api.failsafe.username specified; skipping failsafe account creation")
		return
	}

	log.Debugf("checking to see if we should re-instate the '%s' failsafe account", c.Config.API.Failsafe.Username)
	users, err := c.db.GetAllUsers(&db.UserFilter{Backend: "local"})
	c.MaybeTerminate(err)

	if len(users) > 0 {
		log.Debugf("existing users found in database; skipping failsafe account creation...")
		return
	}

	log.Debugf("creating failsafe account '%s'", c.Config.API.Failsafe.Username)
	user := &db.User{
		Name:    "Administrator",
		Account: c.Config.API.Failsafe.Username,
		Backend: "local",
		SysRole: "admin",
	}
	user.SetPassword(c.Config.API.Failsafe.Password)
	_, err = c.db.CreateUser(user)
	c.MaybeTerminate(err)
}

func (c *Core) ExpireInteractiveSessions() {
	log.Infof("INITIALIZING: expiring all interactive sessions...")
	c.db.ClearExpiredSessions(time.Now())
}

func (c *Core) PrecreateTenants() {
	log.Infof("INITIALIZING: populating SHIELD tenants referenced in auth providers...")

	tenants := make(map[string]bool)
	for _, auth := range c.providers {
		for _, tenant := range auth.ReferencedTenants() {
			if tenant != "SYSTEM" {
				tenants[tenant] = true
			}
		}
	}

	for tenant := range tenants {
		if _, err := c.db.EnsureTenant(tenant); err != nil {
			c.Terminate(fmt.Errorf("unable to pre-create tnant '%s' (referenced in authentication providers): %s", tenant, err))
		}
	}
}

func (c *Core) CleanupLeftoverTasks() {
	log.Infof("INITIALIZING: cleaning up leftover task records...")

	tasks, err := c.db.GetAllTasks(&db.TaskFilter{ForStatus: db.RunningStatus})
	c.MaybeTerminate(err)

	now := time.Now()
	for _, task := range tasks {
		log.Infof("CLEANUP: found task %s in 'running' state at startup; setting to 'failed'", task.UUID)
		if err := c.db.FailTask(task.UUID, now); err != nil {
			log.Errorf("failed to sweep database of running tasks [%s]: %s", task.UUID, err)
			continue
		}

		if task.Op == db.BackupOperation && task.ArchiveUUID != "" {<commit_msg>fix typo<commit_after>package core2

import (
	"fmt"
	"net/http"
	"os"
	"time"

	"github.com/jhunt/go-log"

	"github.com/starkandwayne/shield/core/bus"
	"github.com/starkandwayne/shield/core/fabric"
	"github.com/starkandwayne/shield/core/scheduler"
	"github.com/starkandwayne/shield/core/vault"
	"github.com/starkandwayne/shield/db"
	"github.com/starkandwayne/shield/timespec"
)

func (c Core) Main() {
	/* we need a usable database first */
	c.ConnectToDatabase()
	c.ConfigureMessageBus()

	/* startup cleanup tasks */
	c.CreateFailsafeUser()
	c.ExpireInteractiveSessions()
	c.PrecreateTenants()
	c.CleanupLeftoverTasks()

	/* prepare for operation */
	c.ConnectToVault()
	c.Bind()
	c.StartScheduler()

	log.Infof("INITIALIZATION COMPLETE; entering main loop.")

	fast := time.NewTicker(time.Second * time.Duration(c.Config.Scheduler.FastLoop))
	slow := time.NewTicker(time.Second * time.Duration(c.Config.Scheduler.SlowLoop))
	for {
		select {
		case <-fast.C:
			c.ScheduleBackupTasks()
			c.ScheduleAgentStatusCheckTasks(&db.AgentFilter{Status: "pending"})
			c.RunScheduler()

		case <-slow.C:
			c.CheckArchiveExpiries()
			c.SchedulePurgeTasks()
			c.MarkIrrelevantTasks()
			c.ScheduleAgentStatusCheckTasks(nil)
			c.UpdateGlobalStorageUsage()
			c.UpdateTenantStorageUsage()

			if c.Unlocked() {
				c.PurgeExpiredAPISessions()
				c.ScheduleStorageTestTasks()
			}
		}
	}
}

func (c *Core) ConnectToDatabase() {
	log.Infof("INITIALIZING: connecting to the database...")
	if c.db != nil {
		log.Alertf("ANOMALY: tried to connect to database, but we're already connected...")
		return
	}

	log.Debugf("connecting to database at %s...", c.DataFile("shield.db"))
	db, err := db.Connect(c.DataFile("shield.db"))
	c.MaybeTerminate(err)
	c.db = db

	log.Infof("INITIALIZING: checking database schema...")
	c.MaybeTerminate(c.db.CheckCurrentSchema())

	log.Debugf("connected successfully to database!")
}

func (c *Core) CreateFailsafeUser() {
	log.Infof("INITIALIZING: creating the SHIELD (local auth) failsafe user...")

	if c.Config.API.Failsafe.Username == "" {
		log.Infof("no api.failsafe.username specified; skipping failsafe account creation")
		return
	}

	log.Debugf("checking to see if we should re-instate the '%s' failsafe account", c.Config.API.Failsafe.Username)
	users, err := c.db.GetAllUsers(&db.UserFilter{Backend: "local"})
	c.MaybeTerminate(err)

	if len(users) > 0 {
		log.Debugf("existing users found in database; skipping failsafe account creation...")
		return
	}

	log.Debugf("creating failsafe account '%s'", c.Config.API.Failsafe.Username)
	user := &db.User{
		Name:    "Administrator",
		Account: c.Config.API.Failsafe.Username,
		Backend: "local",
		SysRole: "admin",
	}
	user.SetPassword(c.Config.API.Failsafe.Password)
	_, err = c.db.CreateUser(user)
	c.MaybeTerminate(err)
}

func (c *Core) ExpireInteractiveSessions() {
	log.Infof("INITIALIZING: expiring all interactive sessions...")
	c.db.ClearExpiredSessions(time.Now())
}

func (c *Core) PrecreateTenants() {
	log.Infof("INITIALIZING: populating SHIELD tenants referenced in auth providers...")

	tenants := make(map[string]bool)
	for _, auth := range c.providers {
		for _, tenant := range auth.ReferencedTenants() {
			if tenant != "SYSTEM" {
				tenants[tenant] = true
			}
		}
	}

	for tenant := range tenants {
		if _, err := c.db.EnsureTenant(tenant); err != nil {
			c.Terminate(fmt.Errorf("unable to pre-create tenant '%s' (referenced in authentication providers): %s", tenant, err))
		}
	}
}

func (c *Core) CleanupLeftoverTasks() {
	log.Infof("INITIALIZING: cleaning up leftover task records...")

	tasks, err := c.db.GetAllTasks(&db.TaskFilter{ForStatus: db.RunningStatus})
	c.MaybeTerminate(err)

	now := time.Now()
	for _, task := range tasks {
		log.Infof("CLEANUP: found task %s in 'running' state at startup; setting to 'failed'", task.UUID)
		if err := c.db.FailTask(task.UUID, now); err != nil {
			log.Errorf("failed to sweep database of running tasks [%s]: %s", task.UUID, err)
			continue
		}

		if task.Op == db.BackupOperation && task.ArchiveUUID != "" {
{'fraction_non_alphanumeric': 0.14987714987714987, 'fraction_numerical': 0.0004914004914004914, 'mean_word_length': 4.307040417209909, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1916134', 'n_tokens_mistral': 3146, 'n_tokens_neox': 2903, 'n_words': 599}