<commit_before>package policylib

import (
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"strings"
	"sync"
	"time"

	"../common"
	"../consulLib"
)

type dcData map[string]*common.DC

var (
	dcDataList   dcData
	dcConsulList *consulLib.FederaConsulClient
)

// Policy runs only when there is change in the DC info from the gosspier
// or when there is a change in the Consul KV "Fedra" store
func ListenAllDcDataChn() {
	//<-

}

// ListenDcConsulList will block till the KV client connection is established
// TODO: We will read the Consul DC data and update the global pointer dcConsulList
// This has to be controlled via a mutex.
// But for now we dont lock while reading.
func ListenDcConsulList() {
	//<-
}

// Its tricky to actually make a block call on the consul looking for a KV prefix
// If there is not such KV store we create one with the /Fedra with some data/no data
// we then use the modified index from that store for later block
// config is passed to get the consul KV config directly
func ListenConsulKV(Prefix string, wg *sync.WaitGroup, config *common.ConsulConfig) {

	var waitIndex uint64

	// This is a blocking call since we need to watch the store for any new policy change
	// and notify processNewPloicy

	for {

		//GlobalConsulDCInfo should have valid consul client connections

		if dcConsulList == nil {
			log.Println("dcConsulList global consullist is empty")
			os.Exit(0)
		}

		KVPairs, nextWaitIndex, err := dcConsulList.GetList(Prefix, waitIndex)
		//dcConsulList.GetDataFromLocalKVStore(waitIndex)

		if err != nil {
			log.Println("ListenConsulKV: Watch on local store failes", err)
			//TODO: We just continue for now

		} else {
			log.Println("ListenConsulKV: Got the following data from consul current and new wait index", waitIndex, nextWaitIndex)
			waitIndex = nextWaitIndex
			for _, value := range KVPairs.KVPairs {

				log.Println("ListenConsulKV: key and value ", string(value.Key), string(value.Value))
				processNewPolicy(value.Key, value.Value)

			}

		}

		<-time.After(20 * time.Second)
	}
}

//RuleThreshold
func GetCorrectRuleType(name string) interface{} {

	if strings.Contains(name, "Distance") == true {
		return &RuleDistance{}
	} else if strings.Contains(name, "Threshold") == true {
		return &RuleThreshold{}
	} else {
		log.Println("GetCorrectRuleType: No Rule Type found for", name)
	}
	return nil

}

//processNewPolicy if successfuly processed the policy will be store in a gloable map for
// further processing
// TODO: We need an representaion for the action taken by the policy.

func processNewPolicy(key string, data []byte) error {
	tempPolicy := Policy{}
	err := json.Unmarshal(data, &tempPolicy)

	if err != nil {
		log.Println("processNewPolicy: Unable to unmarshal the processNewPolicy", err)
		return err
	}

	for index, values := range tempPolicy.Rules {

		dummy := FakeJsonRule{}
		ruleType := GetCorrectRuleType(values.Name)
		if ruleType != nil {
			dummy.Content = ruleType
			err := json.Unmarshal(data, &dummy)
			if err != nil {
				log.Println("processNewPolicy: Unable to get the content from the json")
			}
			tempPolicy.Rules[index].Content = dummy.Content<commit_msg>Adding error msg if policy fails<commit_after>package policylib

import (
	"encoding/json"
	"fmt"
	"log"
	"math/rand"
	"os"
	"sort"
	"strings"
	"sync"
	"time"

	"../common"
	"../consulLib"
)

type dcData map[string]*common.DC

var (
	dcDataList   dcData
	dcConsulList *consulLib.FederaConsulClient
)

// Policy runs only when there is change in the DC info from the gosspier
// or when there is a change in the Consul KV "Fedra" store
func ListenAllDcDataChn() {
	//<-

}

// ListenDcConsulList will block till the KV client connection is established
// TODO: We will read the Consul DC data and update the global pointer dcConsulList
// This has to be controlled via a mutex.
// But for now we dont lock while reading.
func ListenDcConsulList() {
	//<-
}

// Its tricky to actually make a block call on the consul looking for a KV prefix
// If there is not such KV store we create one with the /Fedra with some data/no data
// we then use the modified index from that store for later block
// config is passed to get the consul KV config directly
func ListenConsulKV(Prefix string, wg *sync.WaitGroup, config *common.ConsulConfig) {

	var waitIndex uint64

	// This is a blocking call since we need to watch the store for any new policy change
	// and notify processNewPloicy

	for {

		//GlobalConsulDCInfo should have valid consul client connections

		if dcConsulList == nil {
			log.Println("dcConsulList global consullist is empty")
			os.Exit(0)
		}

		KVPairs, nextWaitIndex, err := dcConsulList.GetList(Prefix, waitIndex)
		//dcConsulList.GetDataFromLocalKVStore(waitIndex)

		if err != nil {
			log.Println("ListenConsulKV: Watch on local store failes", err)
			//TODO: We just continue for now

		} else {
			log.Println("ListenConsulKV: Got the following data from consul current and new wait index", waitIndex, nextWaitIndex)
			waitIndex = nextWaitIndex
			for _, value := range KVPairs.KVPairs {

				log.Println("ListenConsulKV: key and value ", string(value.Key), string(value.Value))
				if err := processNewPolicy(value.Key, value.Value); err != nil {
					log.Println("ListenConsulKV: processNewPolicy failed ", err)
				}

			}

		}

		<-time.After(20 * time.Second)
	}
}

//RuleThreshold
func GetCorrectRuleType(name string) interface{} {

	if strings.Contains(name, "Distance") == true {
		return &RuleDistance{}
	} else if strings.Contains(name, "Threshold") == true {
		return &RuleThreshold{}
	} else {
		log.Println("GetCorrectRuleType: No Rule Type found for", name)
	}
	return nil

}

//processNewPolicy if successfuly processed the policy will be store in a gloable map for
// further processing
// TODO: We need an representaion for the action taken by the policy.

func processNewPolicy(key string, data []byte) error {
	tempPolicy := Policy{}
	err := json.Unmarshal(data, &tempPolicy)

	if err != nil {
		log.Println("processNewPolicy: Unable to unmarshal the processNewPolicy", err)
		return err
	}

	for index, values := range tempPolicy.Rules {

		dummy := FakeJsonRule{}
		ruleType := GetCorrectRuleType(values.Name)
		if ruleType != nil {
			dummy.Content = ruleType
			err := json.Unmarshal(data, &dummy)
			if err != nil {
				log.Println("processNewPolicy: Unable to get the content from the json")
				return err
			}
			tempPolicy.Rules[index].Content = dummy.Content
{'fraction_non_alphanumeric': 0.09676425269645608, 'fraction_numerical': 0.0015408320493066256, 'mean_word_length': 3.7763061074319353, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2805661', 'n_tokens_mistral': 2205, 'n_tokens_neox': 2023, 'n_words': 728}