<commit_before>package main

import (
	"flag"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/op/go-logging"
	"github.com/vmihailenco/redis"
	"os"
	"regexp"
)

var (
	path        string
	redisHost   string
	redisPasswd string
	logRegex    string
	redisDB     int64
	log         logging.Logger
)

func init() {
	flag.StringVar(&path, "l", "", "Path to the access log to watch")
	flag.StringVar(&redisHost, "h", "localhost:6379", "Hostname:port Redis")
	flag.StringVar(&redisPasswd, "p", "", "Redis password")
	flag.Int64Var(&redisDB, "d", -1, "Redis DB number to store the data")
	flag.StringVar(&logRegex, "r", "", "PCRE Regex to parse the log. Must return the remote IP on the first capture group")
}

func main() {
	if len(os.Args) < 6 {
		fmt.Fprintln(os.Stderr, "You need arguments")
		flag.Usage()
		os.Exit(10)
	}
	flag.Parse()

	// Seek to the end at the start
	seek := &tail.SeekInfo{
		Offset: 0,
		Whence: 2,
	}

	config := tail.Config{
		ReOpen:    true,
		MustExist: true,
		Follow:    true,
		Location:  seek,
	}

	t, err := tail.TailFile(path, config)
	if err != nil {
		log.Fatalf("Could not tail: %v\n", err)
	}

	client := redis.NewTCPClient(redisHost, redisPasswd, redisDB)
	multi, err := client.MultiClient()
	if err != nil {
		log.Fatalf("Failed to create multiclient: %v", err)
	}
	defer client.Close()
	log.Info("Connected to redis")

	r := regexp.MustCompile(logRegex)
	ipRegex := regexp.MustCompile(`\S+\.\S+\.\S+\.\S+`)

	for line := range t.Lines {
		if matches := r.FindStringSubmatch(line.Text); matches != nil {
			if !ipRegex.MatchString(matches[1]) {
				continue
			}
			if res := client.Ping(); res.Err() != nil {
				log.Warning("Redis not connected, %v, reconnecting", res.Val())
				client = redis.NewTCPClient(redisHost, redisPasswd, redisDB)
				multi, err = client.MultiClient()
				if err != nil {
					log.Fatalf("Failed to create multiclient: %v", err)
				}
			}
			ip := matches[1]
			_, err := multi.Exec(func() {
				multi.ZIncrBy("ipcount_5m", 1, ip)
				multi.ZIncrBy("ipcount_1h", 1, ip)
				multi.ZIncrBy("ipcount_12h", 1, ip)
				multi.ZIncrBy("ipcount_24h", 1, ip)
			})
			if err == redis.Nil {
				log.Warning("Failed to add to set: %v", err)
			}<commit_msg>Adds a secondary hash for expirations (secondary tool)<commit_after>package main

import (
	"flag"
	"fmt"
	"github.com/ActiveState/tail"
	"github.com/op/go-logging"
	"github.com/vmihailenco/redis"
	"os"
	"regexp"
	"time"
)

var (
	path        string
	redisHost   string
	redisPasswd string
	logRegex    string
	redisDB     int64
	log         logging.Logger
)

func init() {
	flag.StringVar(&path, "l", "", "Path to the access log to watch")
	flag.StringVar(&redisHost, "h", "localhost:6379", "Hostname:port Redis")
	flag.StringVar(&redisPasswd, "p", "", "Redis password")
	flag.Int64Var(&redisDB, "d", -1, "Redis DB number to store the data")
	flag.StringVar(&logRegex, "r", "", "PCRE Regex to parse the log. Must return the remote IP on the first capture group")
}

func main() {
	if len(os.Args) < 6 {
		fmt.Fprintln(os.Stderr, "You need arguments")
		flag.Usage()
		os.Exit(10)
	}
	flag.Parse()

	// Seek to the end at the start
	seek := &tail.SeekInfo{
		Offset: 0,
		Whence: 2,
	}

	config := tail.Config{
		ReOpen:    true,
		MustExist: true,
		Follow:    true,
		Location:  seek,
	}

	t, err := tail.TailFile(path, config)
	if err != nil {
		log.Fatalf("Could not tail: %v\n", err)
	}

	client := redis.NewTCPClient(redisHost, redisPasswd, redisDB)
	multi, err := client.MultiClient()
	if err != nil {
		log.Fatalf("Failed to create multiclient: %v", err)
	}
	defer client.Close()
	log.Info("Connected to redis")

	r := regexp.MustCompile(logRegex)
	ipRegex := regexp.MustCompile(`\S+\.\S+\.\S+\.\S+`)

	for line := range t.Lines {
		if matches := r.FindStringSubmatch(line.Text); matches != nil {
			if !ipRegex.MatchString(matches[1]) {
				continue
			}
			if res := client.Ping(); res.Err() != nil {
				log.Warning("Redis not connected, %v, reconnecting", res.Val())
				client = redis.NewTCPClient(redisHost, redisPasswd, redisDB)
				multi, err = client.MultiClient()
				if err != nil {
					log.Fatalf("Failed to create multiclient: %v", err)
				}
			}
			ip := matches[1]
			_, err := multi.Exec(func() {
				multi.ZIncrBy("ipcount_5m", 1, ip)
				multi.HSet("ipcount_h5m", ip, fmt.Sprintf("%d", time.Now().Unix()))
				multi.ZIncrBy("ipcount_1h", 1, ip)
				multi.HSet("ipcount_h1h", ip, fmt.Sprintf("%d", time.Now().Unix()))
				multi.ZIncrBy("ipcount_12h", 1, ip)
				multi.HSet("ipcount_h12h", ip, fmt.Sprintf("%d", time.Now().Unix()))
				multi.ZIncrBy("ipcount_24h", 1, ip)
				multi.HSet("ipcount_h24h", ip, fmt.Sprintf("%d", time.Now().Unix()))
			})
			if err == redis.Nil {
				log.Warning("Failed to add to set: %v", err)
			}
{'fraction_non_alphanumeric': 0.1672605323831482, 'fraction_numerical': 0.012156780549151121, 'mean_word_length': 3.4681647940074907, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7092801', 'n_tokens_mistral': 2028, 'n_tokens_neox': 1837, 'n_words': 440}