<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"
)

func fatalf(msg string, args ...interface{}) {
	fmt.Printf(msg, args...)
	os.Exit(2)
}

// StringArray implements flag.Value interface
type StringArray []string

func (a *StringArray) Set(s string) error {
	msgs := strings.Split(s, ",")
	*a = append(*a, msgs...)
	return nil
}

func (a *StringArray) String() string {
	return strings.Join(*a, ",")
}

func main() {
	host := "localhost:9200"
	indexPrefix := "logstash-"
	msgFields := StringArray{}
	timeField := "@timestamp"
	include := ""
	exclude := ""
	size := 1000
	poll := 1
	useSSL := false
	useSource := false
	showID := false

	flag.StringVar(&host, "host", host, "host and port of elasticsearch")
	flag.StringVar(&indexPrefix, "prefix", indexPrefix, "prefix of log indexes")
	flag.Var(&msgFields, "message", "message fields to display")
	flag.StringVar(&timeField, "timestamp", timeField, "timestap field to sort by")
	flag.StringVar(&include, "include", include, "comma separated list of field:value pairs to include")
	flag.StringVar(&exclude, "exclude", exclude, "comma separated list of field:value pairs to exclude")
	flag.IntVar(&size, "size", size, "number of docs to return per polling interval")
	flag.IntVar(&poll, "poll", poll, "time in seconds to poll for new data from ES")
	flag.BoolVar(&useSSL, "ssl", useSSL, "use https for URI scheme")
	flag.BoolVar(&useSource, "source", useSource, "use _source field to output result")
	flag.BoolVar(&showID, "id", showID, "show _id field")

	flag.Parse()

	// If no message field is explicitly requested we will follow @message
	if len(msgFields) == 0 {
		msgFields = append(msgFields, "@message")
	}

	exFilter := map[string]interface{}{}
	if len(include) > 0 {
		exFilter["bool"] = map[string]interface{}{"must": getTerms(include)}
	}
	if len(exclude) > 0 {
		exFilter["not"] = map[string]interface{}{"or": getTerms(exclude)}
	}
	if len(exFilter) == 0 {
		exFilter["match_all"] = map[string]interface{}{}
	}

	lastTime := time.Now()

	var scheme string
	if useSSL {
		scheme = "https"
	} else {
		scheme = "http"
	}
	rootURL := fmt.Sprintf("%s://%s", scheme, host)

	for {
		resp, err := http.Get(fmt.Sprintf("%s/_status", rootURL))
		if err != nil {
			fatalf("Error contacting Elasticsearch %s: %v", host, err)
		}
		status := map[string]interface{}{}
		if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
			fatalf("Error decoding _status response from Elasticsearch: %v", err)
		}
		resp.Body.Close()
<commit_msg>Fixed query bug when using both include and exclude<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"flag"
	"fmt"
	"io/ioutil"
	"net/http"
	"os"
	"sort"
	"strings"
	"time"
)

func fatalf(msg string, args ...interface{}) {
	fmt.Printf(msg, args...)
	os.Exit(2)
}

// StringArray implements flag.Value interface
type StringArray []string

func (a *StringArray) Set(s string) error {
	msgs := strings.Split(s, ",")
	*a = append(*a, msgs...)
	return nil
}

func (a *StringArray) String() string {
	return strings.Join(*a, ",")
}

func main() {
	host := "localhost:9200"
	indexPrefix := "logstash-"
	msgFields := StringArray{}
	timeField := "@timestamp"
	include := ""
	exclude := ""
	size := 1000
	poll := 1
	useSSL := false
	useSource := false
	showID := false

	flag.StringVar(&host, "host", host, "host and port of elasticsearch")
	flag.StringVar(&indexPrefix, "prefix", indexPrefix, "prefix of log indexes")
	flag.Var(&msgFields, "message", "message fields to display")
	flag.StringVar(&timeField, "timestamp", timeField, "timestap field to sort by")
	flag.StringVar(&include, "include", include, "comma separated list of field:value pairs to include")
	flag.StringVar(&exclude, "exclude", exclude, "comma separated list of field:value pairs to exclude")
	flag.IntVar(&size, "size", size, "number of docs to return per polling interval")
	flag.IntVar(&poll, "poll", poll, "time in seconds to poll for new data from ES")
	flag.BoolVar(&useSSL, "ssl", useSSL, "use https for URI scheme")
	flag.BoolVar(&useSource, "source", useSource, "use _source field to output result")
	flag.BoolVar(&showID, "id", showID, "show _id field")

	flag.Parse()

	// If no message field is explicitly requested we will follow @message
	if len(msgFields) == 0 {
		msgFields = append(msgFields, "@message")
	}

	exFilter := map[string]interface{}{}
	if len(include) == 0 && len(exclude) == 0 {
		exFilter["match_all"] = map[string]interface{}{}
	} else {
		filter := map[string]interface{}{}
		if len(include) > 0 {
			filter["must"] = getTerms(include)
		}
		if len(exclude) > 0 {
			filter["must_not"] = getTerms(exclude)
		}
		exFilter["bool"] = filter
	}

	lastTime := time.Now()

	var scheme string
	if useSSL {
		scheme = "https"
	} else {
		scheme = "http"
	}
	rootURL := fmt.Sprintf("%s://%s", scheme, host)

	for {
		resp, err := http.Get(fmt.Sprintf("%s/_status", rootURL))
		if err != nil {
			fatalf("Error contacting Elasticsearch %s: %v", host, err)
		}
		status := map[string]interface{}{}
		if err := json.NewDecoder(resp.Body).Decode(&status); err != nil {
			fatalf("Error decoding _status response from Elasticsearch: %v", err)
		}
		resp.Body.Close()

{'fraction_non_alphanumeric': 0.16520579268292682, 'fraction_numerical': 0.005525914634146341, 'mean_word_length': 4.496335078534031, 'pattern_counts': {'":': 2, '<': 3, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5763300', 'n_tokens_mistral': 1886, 'n_tokens_neox': 1785, 'n_words': 517}