<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"
)

var (
	record_benchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

// Debug printing.
func (d *Datum) String() string {
	return fmt.Sprintf("%v", *d)
}

func (m *Metric) String() string {
	return fmt.Sprintf("%v", *m)
}

// Sort a slice of metrics.
type Metrics []*Metric

func (ms Metrics) Len() int           { return len(ms) }
func (ms Metrics) Swap(i, j int)      { ms[i], ms[j] = ms[j], ms[i] }
func (ms Metrics) Less(i, j int) bool { return ms[i].Name < ms[j].Name }

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	jsonfile    string // Expected metrics after processing.
}{
	{
		"examples/linecount.em",
		"testdata/linecount.log",
		"testdata/linecount.json",
	},
	{
		"examples/rsyncd.em",
		"testdata/rsyncd.log",
		"testdata/rsyncd.json",
	},
	{
		"examples/sftp.em",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.json",
	},
	{
		"examples/dhcpd.em",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.json",
	},
}

func CompileAndLoad(programfile string) (chan string, string) {
	lines := make(chan string)

	p, err := os.Open(programfile)
	if err != nil {
		return lines, fmt.Sprintf("%s: could not open program file: %s", programfile, err)
	}
	defer p.Close()

	// EmtailDebug = 999 // All the debugging.

	v, errs := Compile(programfile, p)
	if errs != nil {
		return lines, fmt.Sprintf("%s: compile failed: %s", programfile, strings.Join(errs, "\n"))
	}

	e := &engine{}
	e.addVm(v)
	go e.run(lines)
	return lines, ""
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		return
	}
	*syslog_use_current_year = false
	for _, tc := range exampleProgramTests {
		metrics = make([]*Metric, 0)
		name := filepath.Base(tc.programfile)
		if strings.HasSuffix(name, ".em") {
			name = name[:len(name)-3]
		}
		t.Log(name)
		lines, errs := CompileAndLoad(tc.programfile)
		if errs != "" {
			t.Errorf(errs)
			continue
		}

		err := OneShot(tc.logfile, lines)
		if err != nil {
			t.Errorf("Oneshot failed: %s", err)
			continue
		}

		// // Dirty hack to create json files :)
		// {
		// 	j, err := os.Create(tc.jsonfile)
		// 	if err != nil {
		// 		t.Errorf("%s: could not open json file: %s", tc.jsonfile, err)
		// 		continue
		// 	}
		// 	defer j.Close()
		// 	b, err := json.MarshalIndent(metrics, "", "  ")
		// 	if err != nil {
		// 		t.Errorf("couldn't marshall metrics")
		// 		continue
		// 	}
		// 	j.Write(b)
		// }

		j, err := os.Open(tc.jsonfile)
		if err != nil {
			t.Errorf("%s: could not open json file: %s", tc.jsonfile, err)
			continue
		}
		defer j.Close()

		var expected_metrics []*Metric

		d := json.NewDecoder(j)
		err = d.Decode(&expected_metrics)
		if err != nil {
			t.Errorf("%s: could not decode json: %s", tc.jsonfile, err)
			continue
		}
		sort.Sort(Metrics(expected_metrics))
		t.Log("expected")
		// for _, m := range expected_metrics {
		// 	if len(m.Keys) > 0 {
		// 		t.Log(m.Name)
		// 		t.Logf("keys, values: %v, %v", m.Keys, m.Values)
		// 		sort.Sort(m)
		// 		t.Logf("keys, values: %v, %v", m.Keys, m.Values)
		// 	}
		// }

		sort.Sort(Metrics(metrics))
		t.Log("exported")
		// for _, m := range metrics {
		// 	if len(m.Keys) > 0 {
		// 		t.Log(m.Name)
		// 		t.Logf("keys, values: %v, %v", m.Keys, m.Values)
		// 		sort.Sort(m)
		// 		t.Logf("keys, values: %v, %v", m.Keys, m.Values)
		// 	}
		// }

		if !reflect.DeepEqual(expected_metrics, metrics) {
			t.Errorf("%s: metrics don't match.\n\texpected:\n%v\n\treceived:\n%v", tc.programfile, expected_metrics, metrics)
		}
	}
}

// These benchmarks run the testdata logs through the example programs.
func BenchmarkExamplePrograms(b *testing.B) {
	if testing.Short() {
		return
	}
	b.Logf("\n")
	for _, tc := range exampleProgramTests {
		lines, errs := CompileAndLoad(tc.programfile)
		if errs != "" {
			b.Errorf(errs)
		}
		r := testing.Benchmark(func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				b.StopTimer()
				line_count.Set(0)<commit_msg>Remove debug logs and obsolete code.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

package main

import (
	"encoding/csv"
	"encoding/json"
	"flag"
	"fmt"
	"os"
	"path/filepath"
	"reflect"
	"runtime"
	"sort"
	"strconv"
	"strings"
	"testing"
	"time"
)

var (
	record_benchmark = flag.Bool("record_benchmark", false, "Record the benchmark results to 'benchmark_results.csv'.")
)

// Debug printing.
func (d *Datum) String() string {
	return fmt.Sprintf("%v", *d)
}

func (m *Metric) String() string {
	return fmt.Sprintf("%v", *m)
}

// Sort a slice of metrics.
type Metrics []*Metric

func (ms Metrics) Len() int           { return len(ms) }
func (ms Metrics) Swap(i, j int)      { ms[i], ms[j] = ms[j], ms[i] }
func (ms Metrics) Less(i, j int) bool { return ms[i].Name < ms[j].Name }

var exampleProgramTests = []struct {
	programfile string // Example program file.
	logfile     string // Sample log input.
	jsonfile    string // Expected metrics after processing.
}{
	{
		"examples/linecount.em",
		"testdata/linecount.log",
		"testdata/linecount.json",
	},
	{
		"examples/rsyncd.em",
		"testdata/rsyncd.log",
		"testdata/rsyncd.json",
	},
	{
		"examples/sftp.em",
		"testdata/sftp_chroot.log",
		"testdata/sftp_chroot.json",
	},
	{
		"examples/dhcpd.em",
		"testdata/anonymised_dhcpd_log",
		"testdata/anonymised_dhcpd_log.json",
	},
}

func CompileAndLoad(programfile string) (chan string, string) {
	lines := make(chan string)

	p, err := os.Open(programfile)
	if err != nil {
		return lines, fmt.Sprintf("%s: could not open program file: %s", programfile, err)
	}
	defer p.Close()

	// EmtailDebug = 999 // All the debugging.

	v, errs := Compile(programfile, p)
	if errs != nil {
		return lines, fmt.Sprintf("%s: compile failed: %s", programfile, strings.Join(errs, "\n"))
	}

	e := &engine{}
	e.addVm(v)
	go e.run(lines)
	return lines, ""
}

func TestExamplePrograms(t *testing.T) {
	if testing.Short() {
		return
	}
	*syslog_use_current_year = false
	for _, tc := range exampleProgramTests {
		metrics = make([]*Metric, 0)
		name := filepath.Base(tc.programfile)
		if strings.HasSuffix(name, ".em") {
			name = name[:len(name)-3]
		}
		t.Log(name)
		lines, errs := CompileAndLoad(tc.programfile)
		if errs != "" {
			t.Errorf(errs)
			continue
		}

		err := OneShot(tc.logfile, lines)
		if err != nil {
			t.Errorf("Oneshot failed: %s", err)
			continue
		}

		// // Dirty hack to create json files :)
		// {
		// 	j, err := os.Create(tc.jsonfile)
		// 	if err != nil {
		// 		t.Errorf("%s: could not open json file: %s", tc.jsonfile, err)
		// 		continue
		// 	}
		// 	defer j.Close()
		// 	b, err := json.MarshalIndent(metrics, "", "  ")
		// 	if err != nil {
		// 		t.Errorf("couldn't marshall metrics")
		// 		continue
		// 	}
		// 	j.Write(b)
		// }

		j, err := os.Open(tc.jsonfile)
		if err != nil {
			t.Errorf("%s: could not open json file: %s", tc.jsonfile, err)
			continue
		}
		defer j.Close()

		var expected_metrics []*Metric

		d := json.NewDecoder(j)
		err = d.Decode(&expected_metrics)
		if err != nil {
			t.Errorf("%s: could not decode json: %s", tc.jsonfile, err)
			continue
		}
		sort.Sort(Metrics(expected_metrics))
		sort.Sort(Metrics(metrics))
		if !reflect.DeepEqual(expected_metrics, metrics) {
			t.Errorf("%s: metrics don't match.\n\texpected:\n%v\n\treceived:\n%v", tc.programfile, expected_metrics, metrics)
		}
	}
}

// These benchmarks run the testdata logs through the example programs.
func BenchmarkExamplePrograms(b *testing.B) {
	if testing.Short() {
		return
	}
	b.Logf("\n")
	for _, tc := range exampleProgramTests {
		lines, errs := CompileAndLoad(tc.programfile)
		if errs != "" {
			b.Errorf(errs)
		}
		r := testing.Benchmark(func(b *testing.B) {
			for i := 0; i < b.N; i++ {
				b.StopTimer()
				line_count.Set(0)
{'fraction_non_alphanumeric': 0.1720970112689858, 'fraction_numerical': 0.002939735423811857, 'mean_word_length': 3.092731829573935, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4753275', 'n_tokens_mistral': 3438, 'n_tokens_neox': 3167, 'n_words': 738}