<commit_before>package commands

import (
	kvexpress "../kvexpress/"
	"fmt"
	"github.com/spf13/cobra"
	"log"
	"os"
)

var inCmd = &cobra.Command{
	Use:   "in",
	Short: "Put configuration into Consul.",
	Long:  `in is for putting data into a Consul key so that you can write it on another networked node.`,
	Run:   inRun,
}

func inRun(cmd *cobra.Command, args []string) {
	var Direction = "in"
	checkInFlags()

	key_stop := kvexpress.KeyStopPath(KeyInLocation, PrefixLocation, Direction)
	key_data := kvexpress.KeyDataPath(KeyInLocation, PrefixLocation, Direction)
	key_checksum := kvexpress.KeyChecksumPath(KeyInLocation, PrefixLocation, Direction)
	key_updated := kvexpress.KeyUpdatedPath(KeyInLocation, PrefixLocation, Direction)

	compare_file := kvexpress.CompareFilename(FiletoRead)
	last_file := kvexpress.LastFilename(FiletoRead)

	StopKeyData := kvexpress.Get(key_stop, ConsulServer, Token, Direction)

	if StopKeyData != "" {
		log.Print(Direction, ": Stop Key is present - stopping.")
		os.Exit(1)
	} else {
		log.Print(Direction, ": Stop Key is NOT present - continuing.")
	}

	// Read the file - if it's to be sorted - then make sure to sort.
	// TODO: Do we need to uniq as well?
	file_string := kvexpress.ReadFile(FiletoRead)
	log.Print(Direction, ": file_string='", file_string, "'")

	if Sorted {
		file_string = kvexpress.SortFile(file_string)
	}

	// Is it long enough?
	longEnough := kvexpress.LengthCheck(file_string, MinFileLength, Direction)

	if !longEnough {
		log.Print(Direction, ": File is NOT long enough. Stopping.")
		os.Exit(1)
	}

	// Write the .compare file.
	kvexpress.WriteFile(file_string, compare_file, FilePermissions, Direction)

	// Check for the .last file - touch if it doesn't exist.
	kvexpress.CheckLastFile(last_file, FilePermissions)

	// Read compare and last files into string.
	compare_data := kvexpress.ReadFile(compare_file)
	last_data := kvexpress.ReadFile(last_file)

	if compare_data != "" && last_data != "" {
		log.Print(Direction, ": We have data - let's do the thing.")

		// Get SHA256 values for each string.
		compare_checksum := kvexpress.ComputeChecksum(compare_data, Direction)
		last_checksum := kvexpress.ComputeChecksum(last_data, Direction)

		// If they're different - let's update things.
		if compare_checksum != last_checksum {
			log.Print(Direction, ": file checksums are different - let's update some stuff!")

			// Diff the file data.
			// html_diff := kvexpress.HTMLDiff(last_data, compare_data)

			// TODO: To be removed.
			// fmt.Printf("%v", html_diff)

			// Get the checksum from Consul.
			current_checksum := kvexpress.Get(key_checksum, ConsulServer, Token, Direction)

			if current_checksum != compare_checksum {
				log.Print(Direction, ": current and previous Consul checksum are different - let's update the KV store.")
				saved := kvexpress.Set(key_data, compare_data, ConsulServer, Token, Direction)
				if saved {
					log.Print(Direction, ": key_data='", key_data, "' saved='true' size='", len(compare_data), "'")
					kvexpress.Set(key_checksum, compare_checksum, ConsulServer, Token, Direction)
					kvexpress.Set(key_updated, kvexpress.ReturnCurrentUTC(), ConsulServer, Token, Direction)

					// Copy the compare_data to the .last file.
					kvexpress.WriteFile(compare_data, last_file, FilePermissions, Direction)
				} else {
					log.Print(Direction, ": key_data='", key_data, "' saved='false'")
					os.Exit(1)
				}
			} else {<commit_msg>Remove `updated` key - it's not necessary and doesn't add anything.<commit_after>package commands

import (
	kvexpress "../kvexpress/"
	"fmt"
	"github.com/spf13/cobra"
	"log"
	"os"
)

var inCmd = &cobra.Command{
	Use:   "in",
	Short: "Put configuration into Consul.",
	Long:  `in is for putting data into a Consul key so that you can write it on another networked node.`,
	Run:   inRun,
}

func inRun(cmd *cobra.Command, args []string) {
	var Direction = "in"
	checkInFlags()

	key_stop := kvexpress.KeyStopPath(KeyInLocation, PrefixLocation, Direction)
	key_data := kvexpress.KeyDataPath(KeyInLocation, PrefixLocation, Direction)
	key_checksum := kvexpress.KeyChecksumPath(KeyInLocation, PrefixLocation, Direction)

	compare_file := kvexpress.CompareFilename(FiletoRead)
	last_file := kvexpress.LastFilename(FiletoRead)

	StopKeyData := kvexpress.Get(key_stop, ConsulServer, Token, Direction)

	if StopKeyData != "" {
		log.Print(Direction, ": Stop Key is present - stopping.")
		os.Exit(1)
	} else {
		log.Print(Direction, ": Stop Key is NOT present - continuing.")
	}

	// Read the file - if it's to be sorted - then make sure to sort.
	// TODO: Do we need to uniq as well?
	file_string := kvexpress.ReadFile(FiletoRead)
	log.Print(Direction, ": file_string='", file_string, "'")

	if Sorted {
		file_string = kvexpress.SortFile(file_string)
	}

	// Is it long enough?
	longEnough := kvexpress.LengthCheck(file_string, MinFileLength, Direction)

	if !longEnough {
		log.Print(Direction, ": File is NOT long enough. Stopping.")
		os.Exit(1)
	}

	// Write the .compare file.
	kvexpress.WriteFile(file_string, compare_file, FilePermissions, Direction)

	// Check for the .last file - touch if it doesn't exist.
	kvexpress.CheckLastFile(last_file, FilePermissions)

	// Read compare and last files into string.
	compare_data := kvexpress.ReadFile(compare_file)
	last_data := kvexpress.ReadFile(last_file)

	if compare_data != "" && last_data != "" {
		log.Print(Direction, ": We have data - let's do the thing.")

		// Get SHA256 values for each string.
		compare_checksum := kvexpress.ComputeChecksum(compare_data, Direction)
		last_checksum := kvexpress.ComputeChecksum(last_data, Direction)

		// If they're different - let's update things.
		if compare_checksum != last_checksum {
			log.Print(Direction, ": file checksums are different - let's update some stuff!")

			// Diff the file data.
			// html_diff := kvexpress.HTMLDiff(last_data, compare_data)

			// TODO: To be removed.
			// fmt.Printf("%v", html_diff)

			// Get the checksum from Consul.
			current_checksum := kvexpress.Get(key_checksum, ConsulServer, Token, Direction)

			if current_checksum != compare_checksum {
				log.Print(Direction, ": current and previous Consul checksum are different - let's update the KV store.")
				saved := kvexpress.Set(key_data, compare_data, ConsulServer, Token, Direction)
				if saved {
					log.Print(Direction, ": key_data='", key_data, "' saved='true' size='", len(compare_data), "'")
					kvexpress.Set(key_checksum, compare_checksum, ConsulServer, Token, Direction)

					// Copy the compare_data to the .last file.
					kvexpress.WriteFile(compare_data, last_file, FilePermissions, Direction)
				} else {
					log.Print(Direction, ": key_data='", key_data, "' saved='false'")
					os.Exit(1)
				}
			} else {
{'fraction_non_alphanumeric': 0.12095702259636686, 'fraction_numerical': 0.0023630187564613794, 'mean_word_length': 4.217257318952234, 'pattern_counts': {'":': 18, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453161', 'n_tokens_mistral': 2468, 'n_tokens_neox': 2234, 'n_words': 586}