<commit_before>package main

import (
	"io/ioutil"
	"encoding/json"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Policy struct {
	MID				 bson.ObjectId 				 `bson:"_id,omitempty" json:"id"`
	ID  			 string 				  	 `bson:"id,omitempty" json:"id"`
	OrgID			 string						 `bson:"org_id" json:"org_id"`
	Rate             float64                     `bson:"rate" json:"rate"`
	Per              float64                     `bson:"per" json:"per"`
	QuotaMax         int64                       `bson:"quota_max" json:"quota_max"`
	QuotaRenewalRate int64                       `bson:"quota_renewal_rate" json:"quota_renewal_rate"`
	AccessRights     map[string]AccessDefinition `bson:"access_rights" json:"access_rights"`
	HMACEnabled      bool        				 `bson:"hmac_enabled" json:"hmac_enabled"`
	Active bool `bson:"active" json:"active"`
}

func LoadPoliciesFromFile(filePath string) map[string]Policy {
	policies := make(map[string]Policy)
	
	policyConfig, err := ioutil.ReadFile(filePath)
	if err != nil {
		log.Error("Couldn't load policy file: ", err)
		return policies
	} 

	mErr := json.Unmarshal(policyConfig, &policies)
	if mErr != nil {
		log.Error("Couldn't unmarshal policies: ", mErr)
	}
	
	return policies
}

// LoadPoliciesFromMongo will connect and download POlicies from a Mongo DB instance.
func LoadPoliciesFromMongo(collectionName string) map[string]Policy {
	dbPolicyList := make([]Policy, 0)
	policies := make(map[string]Policy)

	dbSession, dErr := mgo.Dial(config.AnalyticsConfig.MongoURL)
	if dErr != nil {
		log.Error("Mongo connection failed:", dErr)
	}
	
	log.Debug("Searching in collection: ", collectionName)
	policyCollection := dbSession.DB("").C(collectionName)

	search := bson.M{
		"active": true,
	}

	mongoErr := policyCollection.Find(search).All(&dbPolicyList)
<commit_msg>Minor policy update [ci skip]<commit_after>package main

import (
	"io/ioutil"
	"encoding/json"
	"labix.org/v2/mgo"
	"labix.org/v2/mgo/bson"
)

type Policy struct {
	MID				 bson.ObjectId 				 `bson:"_id,omitempty" json:"id"`
	ID  			 string 				  	 `bson:"id,omitempty" json:"id"`
	OrgID			 string						 `bson:"org_id" json:"org_id"`
	Rate             float64                     `bson:"rate" json:"rate"`
	Per              float64                     `bson:"per" json:"per"`
	QuotaMax         int64                       `bson:"quota_max" json:"quota_max"`
	QuotaRenewalRate int64                       `bson:"quota_renewal_rate" json:"quota_renewal_rate"`
	AccessRights     map[string]AccessDefinition `bson:"access_rights" json:"access_rights"`
	HMACEnabled      bool        				 `bson:"hmac_enabled" json:"hmac_enabled"`
	Active bool `bson:"active" json:"active"`
}

func LoadPoliciesFromFile(filePath string) map[string]Policy {
	policies := make(map[string]Policy)
	
	policyConfig, err := ioutil.ReadFile(filePath)
	if err != nil {
		log.Error("Couldn't load policy file: ", err)
		return policies
	} 

	mErr := json.Unmarshal(policyConfig, &policies)
	if mErr != nil {
		log.Error("Couldn't unmarshal policies: ", mErr)
	}
	
	return policies
}

// LoadPoliciesFromMongo will connect and download POlicies from a Mongo DB instance.
func LoadPoliciesFromMongo(collectionName string) map[string]Policy {
	dbPolicyList := make([]Policy, 0)
	policies := make(map[string]Policy)

	dbSession, dErr := mgo.Dial(config.AnalyticsConfig.MongoURL)
	if dErr != nil {
		log.Error("Mongo connection failed:", dErr)
	}
	
	log.Debug("Searching in collection: ", collectionName)
	policyCollection := dbSession.DB("").C(collectionName)

	 //TODO: Should check by OrgID
	log.Warning("POLICIES NOT BEING CHECKED BY ORGID")
	search := bson.M{
		"active": true,
	}

	mongoErr := policyCollection.Find(search).All(&dbPolicyList)

{'fraction_non_alphanumeric': 0.12775093934514226, 'fraction_numerical': 0.005904455179817499, 'mean_word_length': 3.468824940047962, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7162622', 'n_tokens_mistral': 1372, 'n_tokens_neox': 1268, 'n_words': 261}