<commit_before>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"text/template"
	"time"
)

func doElkVerifications(config Config) []string {
	var errors []string

	for _, c := range config.ElkConfiguration {
		errors = append(errors, doElkVerification(c)...)
	}

	return errors
}

type ElkTemplateData struct {
	Date    string
	Query   string
	Minutes string
}

func doElkVerification(config ElkConfiguration) []string {
	const elkTemplate = `curl -XPOST localhost:9200/logstash-{{.Date}}/logs/_search -d '{
  "query": {
    "filtered": {
      "query": {
        "query_string": {
          "query": "{{.Query}}"
        }
      },
      "filter": {
        "bool": {
          "must": [
            {
              "range": {
                "@timestamp": {
                  "gte": "now-{{.Minutes}}m"
                }
              }
            }
          ],
          "must_not": []
        }
      }
    }
  },
  "size": 500,
  "sort": {
    "@timestamp": "desc"
  },
  "fields": [
    "_source"
  ],
  "script_fields": {},
  "fielddata_fields": [
    "timestamp",
    "@timestamp"
  ]
}'
`
	var errors []string

	tmpl, err := template.New("elk").Parse(elkTemplate)
	if err != nil {
		errors = append(errors, fmt.Sprintf("Failed to parse elk template: %s\n", fmt.Sprint(err)))
		return errors
	}

	indexes := elkIndexToUse(time.Now().UTC(), config.Minutes)

	// if multiple indexes that will result in multiple calls to logstash
	// i.e. the results might be a combination of a query against the pre-midnight index and the
	// post-midnight index (as logstash does index rotation at midnight utc)

	var outputs []string
	for _, index := range indexes {
		templateData := ElkTemplateData{index,
			template.JSEscapeString(config.Query), fmt.Sprintf("%d", config.Minutes)}

		var b bytes.Buffer
		err = tmpl.Execute(&b, templateData)
		if err != nil {
			errors = append(errors, fmt.Sprintf("Failed to parse elk template: %s\n", fmt.Sprint(err)))
			return errors
		}

		cmd := fmt.Sprintf("docker exec elk %s", b.String())
		o, err := exec.Command("bash", "-c", cmd).Output()
		if err != nil {
			errors = append(errors, fmt.Sprintf("Failed to run docker command: %s\n", fmt.Sprint(err)))
		}

		outputs = append(outputs, string(o))
	}

	if config.MatchesEqual != nil {
		e := verifyElkExpectedNoOfMatches(outputs, *config.MatchesEqual)
		errors = append(errors, e...)
	} else {
		e := verifyElkAtLeastNoOfMatches(outputs, *config.MatchesAtLeast)
		errors = append(errors, e...)<commit_msg>Stop processing if docker command fails and log what it tried to run.<commit_after>package main

import (
	"bytes"
	"encoding/json"
	"fmt"
	"os/exec"
	"text/template"
	"time"
)

func doElkVerifications(config Config) []string {
	var errors []string

	for _, c := range config.ElkConfiguration {
		errors = append(errors, doElkVerification(c)...)
	}

	return errors
}

type ElkTemplateData struct {
	Date    string
	Query   string
	Minutes string
}

func doElkVerification(config ElkConfiguration) []string {
	const elkTemplate = `curl -XPOST localhost:9200/logstash-{{.Date}}/logs/_search -d '{
  "query": {
    "filtered": {
      "query": {
        "query_string": {
          "query": "{{.Query}}"
        }
      },
      "filter": {
        "bool": {
          "must": [
            {
              "range": {
                "@timestamp": {
                  "gte": "now-{{.Minutes}}m"
                }
              }
            }
          ],
          "must_not": []
        }
      }
    }
  },
  "size": 500,
  "sort": {
    "@timestamp": "desc"
  },
  "fields": [
    "_source"
  ],
  "script_fields": {},
  "fielddata_fields": [
    "timestamp",
    "@timestamp"
  ]
}'
`
	var errors []string

	tmpl, err := template.New("elk").Parse(elkTemplate)
	if err != nil {
		errors = append(errors, fmt.Sprintf("Failed to parse elk template: %s\n", fmt.Sprint(err)))
		return errors
	}

	indexes := elkIndexToUse(time.Now().UTC(), config.Minutes)

	// if multiple indexes that will result in multiple calls to logstash
	// i.e. the results might be a combination of a query against the pre-midnight index and the
	// post-midnight index (as logstash does index rotation at midnight utc)

	var outputs []string
	for _, index := range indexes {
		templateData := ElkTemplateData{index,
			template.JSEscapeString(config.Query), fmt.Sprintf("%d", config.Minutes)}

		var b bytes.Buffer
		err = tmpl.Execute(&b, templateData)
		if err != nil {
			errors = append(errors, fmt.Sprintf("Failed to parse elk template: %s\n", fmt.Sprint(err)))
			return errors
		}

		cmd := fmt.Sprintf("docker exec elk %s", b.String())
		o, err := exec.Command("bash", "-c", cmd).Output()
		if err != nil {
			errors = append(errors, fmt.Sprintf("Failed to run docker command: %s\n\nReturn code: %s\n", cmd, fmt.Sprint(err)))
			return errors
		}

		outputs = append(outputs, string(o))
	}

	if config.MatchesEqual != nil {
		e := verifyElkExpectedNoOfMatches(outputs, *config.MatchesEqual)
		errors = append(errors, e...)
	} else {
		e := verifyElkAtLeastNoOfMatches(outputs, *config.MatchesAtLeast)
		errors = append(errors, e...)
{'fraction_non_alphanumeric': 0.15084778795556422, 'fraction_numerical': 0.002728512960436562, 'mean_word_length': 2.7350800582241632, 'pattern_counts': {'":': 36, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6385386', 'n_tokens_mistral': 1797, 'n_tokens_neox': 1695, 'n_words': 422}