<commit_before>package logyard

import (
	"encoding/json"
	"github.com/ActiveState/doozer"
	"github.com/ActiveState/doozerconfig"
	"github.com/ActiveState/log"
)

type logyardConfig struct {
	Drains      map[string]string `doozer:"drains"`
	RetryLimits map[string]string `doozer:"retrylimits"`
	Doozer      *doozer.Conn
	Rev         int64                     // Doozer revision this config struct corresponds to
	Ch          chan *doozerconfig.Change // Doozer changes to the Drains map
}

// Logyard configuration object tied to doozer config
var Config *logyardConfig

const DOOZER_PREFIX = "/proc/logyard/config/"

var doozerCfg *doozerconfig.DoozerConfig

// Init initializes logyard with the given config entry point in
// doozer.
func Init(conn *doozer.Conn, rev int64, monitor bool) {
	Config = new(logyardConfig)
	Config.Drains = make(map[string]string)
	Config.RetryLimits = make(map[string]string)
	Config.Ch = make(chan *doozerconfig.Change)
	Config.Rev = rev
	Config.Doozer = conn
	doozerCfg = doozerconfig.New(conn, rev, Config, DOOZER_PREFIX)
	err := doozerCfg.Load()
	if err != nil {
		log.Fatal(err)
	}

	if !monitor {
		return
	}

	// Monitor drain config changes in doozer
	go doozerCfg.Monitor(DOOZER_PREFIX+"drains/*", func(change *doozerconfig.Change, err error) {
		if err != nil {
			// don't bring down the entire logyard, because we want
			// the existing drains to continue to function despite an
			// error in monitoring config changes.
			log.Errorf("Unable to process drain config change in doozer: %s", err)
			return
		}
		log.Infof("Detected change in doozer drains config: %s", change.Key)
		// Config.Rev = TODO
		if change.FieldName == "Drains" {
			switch change.Type {
			case doozerconfig.DELETE, doozerconfig.SET:
				Config.Ch <- change
			}
		}
	})
}

// DeleteDrain deletes the drain from doozer tree
func (Config *logyardConfig) DeleteDrain(name string) error {
	err := Config.Doozer.Del(DOOZER_PREFIX+"drains/"+name, Config.Rev)
	if err != nil {
		return err
	}
	return nil
}
<commit_msg>allow deletion without rev<commit_after>package logyard

import (
	"encoding/json"
	"github.com/ActiveState/doozer"
	"github.com/ActiveState/doozerconfig"
	"github.com/ActiveState/log"
)

type logyardConfig struct {
	Drains      map[string]string `doozer:"drains"`
	RetryLimits map[string]string `doozer:"retrylimits"`
	Doozer      *doozer.Conn
	Rev         int64                     // Doozer revision this config struct corresponds to
	Ch          chan *doozerconfig.Change // Doozer changes to the Drains map
}

// Logyard configuration object tied to doozer config
var Config *logyardConfig

const DOOZER_PREFIX = "/proc/logyard/config/"

var doozerCfg *doozerconfig.DoozerConfig

// Init initializes logyard with the given config entry point in
// doozer.
func Init(conn *doozer.Conn, rev int64, monitor bool) {
	Config = new(logyardConfig)
	Config.Drains = make(map[string]string)
	Config.RetryLimits = make(map[string]string)
	Config.Ch = make(chan *doozerconfig.Change)
	Config.Rev = rev
	Config.Doozer = conn
	doozerCfg = doozerconfig.New(conn, rev, Config, DOOZER_PREFIX)
	err := doozerCfg.Load()
	if err != nil {
		log.Fatal(err)
	}

	if !monitor {
		return
	}

	// Monitor drain config changes in doozer
	go doozerCfg.Monitor(DOOZER_PREFIX+"drains/*", func(change *doozerconfig.Change, err error) {
		if err != nil {
			// don't bring down the entire logyard, because we want
			// the existing drains to continue to function despite an
			// error in monitoring config changes.
			log.Errorf("Unable to process drain config change in doozer: %s", err)
			return
		}
		log.Infof("Detected change in doozer drains config: %s", change.Key)
		// Config.Rev = TODO
		if change.FieldName == "Drains" {
			switch change.Type {
			case doozerconfig.DELETE, doozerconfig.SET:
				Config.Ch <- change
			}
		}
	})
}

// DeleteDrain deletes the drain from doozer tree
func (Config *logyardConfig) DeleteDrain(name string) error {
	err := Config.Doozer.Del(DOOZER_PREFIX+"drains/"+name, -1)
	if err != nil {
		return err
	}
	return nil
}

{'fraction_non_alphanumeric': 0.10508224895654308, 'fraction_numerical': 0.00220967345936656, 'mean_word_length': 3.6506849315068495, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1909408', 'n_tokens_mistral': 1460, 'n_tokens_neox': 1388, 'n_words': 381}