<commit_before>package backup

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/pshima/consul-snapshot/config"
	"github.com/pshima/consul-snapshot/consul"
	"github.com/pshima/consul-snapshot/health"
)

// Backup is the backup itself including configuration and data
type Backup struct {
	StartTime      int64
	JSONData       []byte
	LocalFileName  string
	LocalFilePath  string
	RemoteFilePath string
	Config         config.Config
	Client         *consul.Consul
}

// Runner is the main runner for a backup
func (b *Backup) Runner(t string) int {
	// Start up the http server health checks
	go health.StartServer()

	if t == "test" {
		b.doWork(t)
	} else {
		log.Printf("[DEBUG] Backup starting on interval: %v", b.Config.BackupInterval)
		ticker := time.NewTicker(b.Config.BackupInterval)
		for range ticker.C {
			b.doWork(t)
		}
	}
	return 0
}

func (b *Backup) doWork(t string) {
	// Loop over and over at interval time.
	b.StartTime = time.Now().Unix()

	if t == "test" {
		b.LocalFileName = "acceptancetest.gz"
	}

	startString := fmt.Sprintf("%v", b.StartTime)

	log.Printf("[INFO] Starting Backup At: %s", startString)
	log.Print("[INFO] Listing keys from consul")

	b.Client.ListKeys()
	log.Print("[INFO] Converting keys to JSON")
	b.KeysToJSON()
	log.Print("[INFO] Writing Local Backup File")
	b.writeBackupLocal()
	if t != "test" {
		log.Print("[INFO] Writing Backup to Remote File")
		b.writeBackupRemote()
	} else {
		log.Print("[INFO] Skipping remote backup during testing")
	}
	if t != "test" {
		log.Print("[INFO] Running post processing")
		b.postProcess()
	} else {
		log.Print("[INFO] Skipping post processing during testing")
	}
	log.Print("[INFO] Backup completed successfully")
}

// KeysToJSON used to marshall the data and put it on a Backup object
func (b *Backup) KeysToJSON() {
	jsonData, err := json.Marshal(b.Client.KeyData)
	if err != nil {
		log.Fatalf("[ERR] Could not encode keys to json!: %v", err)
	}
	b.JSONData = jsonData
}<commit_msg>Add count of keys to log output<commit_after>package backup

import (
	"bytes"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"time"

	"github.com/aws/aws-sdk-go/aws"
	"github.com/aws/aws-sdk-go/aws/session"
	"github.com/aws/aws-sdk-go/service/s3/s3manager"
	consulapi "github.com/hashicorp/consul/api"
	"github.com/pshima/consul-snapshot/config"
	"github.com/pshima/consul-snapshot/consul"
	"github.com/pshima/consul-snapshot/health"
)

// Backup is the backup itself including configuration and data
type Backup struct {
	StartTime      int64
	JSONData       []byte
	LocalFileName  string
	LocalFilePath  string
	RemoteFilePath string
	Config         config.Config
	Client         *consul.Consul
}

// Runner is the main runner for a backup
func (b *Backup) Runner(t string) int {
	// Start up the http server health checks
	go health.StartServer()

	if t == "test" {
		b.doWork(t)
	} else {
		log.Printf("[DEBUG] Backup starting on interval: %v", b.Config.BackupInterval)
		ticker := time.NewTicker(b.Config.BackupInterval)
		for range ticker.C {
			b.doWork(t)
		}
	}
	return 0
}

func (b *Backup) doWork(t string) {
	// Loop over and over at interval time.
	b.StartTime = time.Now().Unix()

	if t == "test" {
		b.LocalFileName = "acceptancetest.gz"
	}

	startString := fmt.Sprintf("%v", b.StartTime)

	log.Printf("[INFO] Starting Backup At: %s", startString)
	log.Print("[INFO] Listing keys from consul")

	b.Client.ListKeys()
	log.Printf("[INFO] Converting %v keys to JSON", b.Client.KeyDataLen)
	b.KeysToJSON()
	log.Print("[INFO] Writing Local Backup File")
	b.writeBackupLocal()
	if t != "test" {
		log.Print("[INFO] Writing Backup to Remote File")
		b.writeBackupRemote()
	} else {
		log.Print("[INFO] Skipping remote backup during testing")
	}
	if t != "test" {
		log.Print("[INFO] Running post processing")
		b.postProcess()
	} else {
		log.Print("[INFO] Skipping post processing during testing")
	}
	log.Print("[INFO] Backup completed successfully")
}

// KeysToJSON used to marshall the data and put it on a Backup object
func (b *Backup) KeysToJSON() {
	jsonData, err := json.Marshal(b.Client.KeyData)
	if err != nil {
		log.Fatalf("[ERR] Could not encode keys to json!: %v", err)
	}
	b.JSONData = jsonData
}
{'fraction_non_alphanumeric': 0.13954535223947784, 'fraction_numerical': 0.0022507314877335135, 'mean_word_length': 3.773361976369495, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1018182', 'n_tokens_mistral': 1653, 'n_tokens_neox': 1646, 'n_words': 416}