<commit_before>package commands

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type ConsulEvent struct {
	Id            string `json:"ID"`
	Name          string `json:"Name"`
	Payload       string `json:"Payload,omitempty"`
	NodeFilter    string `json:"NodeFilter,omitempty"`
	ServiceFilter string `json:"ServiceFilter"`
	TagFilter     string `json:"TagFilter"`
	Version       int    `json:"Version"`
	LTime         int    `json:"LTime"`
}

func runCommand(command string) bool {
	parts := strings.Fields(command)
	cli := parts[0]
	args := parts[1:len(parts)]
	cmd := exec.Command(cli, args...)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		fmt.Println("exec='error' message='%v'", err)
		return false
	} else {
		return true
	}
}

func getHostname() string {
	hostname, _ := os.Hostname()
	return hostname
}

func createKey(event string) string {
	hostname := getHostname()
	return fmt.Sprintf("sifter/%s/%s", event, hostname)
}

func readStdin() string {
	bytes, _ := ioutil.ReadAll(os.Stdin)
	stdin := string(bytes)
	if stdin == "" || stdin == "[]\n" || stdin == "\n" {
		return ""
	} else {
		// TODO: Yes this is a gross hack and only works if
		// there is a single event in the payload.
		stdin = strings.TrimPrefix(stdin, "[")
		stdin = strings.TrimSuffix(stdin, "]\n")
		return stdin
	}
}

func decodeStdin(data string) (string, int64) {
	var events ConsulEvent
	err := json.Unmarshal([]byte(data), &events)
	if err != nil {
		Log(fmt.Sprintf("error: %s", data), "info")
		os.Exit(1)
	}
	name := string(events.Name)
	lTime := int64(events.LTime)
	Log(fmt.Sprintf("decoded event='%s' ltime='%d'", name, lTime), "info")
	return name, lTime
}
<commit_msg>Found https://coderwall.com/p/4c2zig/decode-top-level-json-array-into-a-slice-of-structs-in-golang - which helped me figure out how to unmarshal a JSON array. Closes #2.<commit_after>package commands

import (
	"bytes"
	"encoding/json"
	"fmt"
	"io/ioutil"
	"os"
	"os/exec"
	"strings"
)

type ConsulEvent struct {
	Id            string `json:"ID"`
	Name          string `json:"Name"`
	Payload       string `json:"Payload,omitempty"`
	NodeFilter    string `json:"NodeFilter,omitempty"`
	ServiceFilter string `json:"ServiceFilter"`
	TagFilter     string `json:"TagFilter"`
	Version       int    `json:"Version"`
	LTime         int    `json:"LTime"`
}

func runCommand(command string) bool {
	parts := strings.Fields(command)
	cli := parts[0]
	args := parts[1:len(parts)]
	cmd := exec.Command(cli, args...)
	var out bytes.Buffer
	cmd.Stdout = &out
	err := cmd.Run()
	if err != nil {
		fmt.Println("exec='error' message='%v'", err)
		return false
	} else {
		return true
	}
}

func getHostname() string {
	hostname, _ := os.Hostname()
	return hostname
}

func createKey(event string) string {
	hostname := getHostname()
	return fmt.Sprintf("sifter/%s/%s", event, hostname)
}

func readStdin() string {
	bytes, _ := ioutil.ReadAll(os.Stdin)
	stdin := string(bytes)
	if stdin == "" || stdin == "[]\n" || stdin == "\n" {
		return ""
	} else {
		return stdin
	}
}

func decodeStdin(data string) (string, int64) {
	events := make([]ConsulEvent, 0)
	err := json.Unmarshal([]byte(data), &events)
	if err != nil {
		Log(fmt.Sprintf("error: %s", data), "info")
		os.Exit(1)
	}
	var name = ""
	var lTime = int64(0)
	for _, event := range events {
		name = string(event.Name)
		if int64(event.LTime) > lTime {
			lTime = int64(event.LTime)
		}
	}
	Log(fmt.Sprintf("decoded event='%s' ltime='%d'", name, lTime), "info")
	return name, lTime
}

{'fraction_non_alphanumeric': 0.16610455311973019, 'fraction_numerical': 0.006464305789769534, 'mean_word_length': 3.6583769633507854, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 1, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6494216', 'n_tokens_mistral': 1339, 'n_tokens_neox': 1307, 'n_words': 318}