<commit_before>package main

// jsonselect is a command-line tool to apply JSONSelect filters to
// JSON through stdin, line by line.
//
// You can use as (eg. filtering Mixpanel-like event data):
//
// Extract the `event` prop, one by line:
//
//     cat jsonfile | jsonselect .event
//
// Extract two lines for each incoming line, one is the `event` property, the other the JSONPath equivalent to `.properties.os_name`
//
//     cat jsonfile | jsonselect .event ".properties .os_name"
//
// Same thing, on a single line, separated by \t characters:
//
//     cat jsonfile | jsonselect -s .event ".properties .os_name"
//
// Nicely indented properties dictionary, prefixed with the `event` as quoted (-q) text:
//
//     cat jsonfile | jsonselect -q -i .event .properties
//



import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"

	jsonselect "github.com/coddingtonbear/go-jsonselect"
)

func main() {
	var singleLine bool
	var quotedStrings bool
	var indent bool

	flag.BoolVar(&singleLine, "s", false, "Put things on a single line")
	flag.BoolVar(&quotedStrings, "q", false, "Keep strings quoted instead of unquoting them")
	flag.BoolVar(&indent, "i", false, "Nicely indent any JSON output")
	flag.Parse()

	errored := false

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		elements, err := elementsForAllPatterns(scanner.Text(), flag.Args())
		if err != nil {
			log.Println("Error:", err)
			return
		}

		for i, el := range elements {
			if i > 0 {
				if singleLine {
					fmt.Print("\t")
				} else {
					fmt.Print("\n")
				}
			}

			stringElement, isString := el.(string)
			if !quotedStrings && isString {
				fmt.Print(stringElement)
			} else {
				var out []byte
				if indent {
					out, err = json.MarshalIndent(el, "", "  ")
				} else {
					out, err = json.Marshal(el)
				}
				if err != nil {
					log.Printf("Error marshalling %v: %s\n", el, err)
					errored = true
				}
<commit_msg>Added fallback to `:root` node if none specified.<commit_after>package main

// jsonselect is a command-line tool to apply JSONSelect filters to
// JSON through stdin, line by line.
//
// You can use as (eg. filtering Mixpanel-like event data):
//
// Extract the `event` prop, one by line:
//
//     cat jsonfile | jsonselect .event
//
// Extract two lines for each incoming line, one is the `event` property, the other the JSONPath equivalent to `.properties.os_name`
//
//     cat jsonfile | jsonselect .event ".properties .os_name"
//
// Same thing, on a single line, separated by \t characters:
//
//     cat jsonfile | jsonselect -s .event ".properties .os_name"
//
// Nicely indented properties dictionary, prefixed with the `event` as quoted (-q) text:
//
//     cat jsonfile | jsonselect -q -i .event .properties
//
//
// Merely running `cat jsonfile | jsonselect -i` will display `:root` by default.

import (
	"bufio"
	"encoding/json"
	"flag"
	"fmt"
	"log"
	"os"

	jsonselect "github.com/coddingtonbear/go-jsonselect"
)

func main() {
	var singleLine bool
	var quotedStrings bool
	var indent bool

	flag.BoolVar(&singleLine, "s", false, "Put things on a single line")
	flag.BoolVar(&quotedStrings, "q", false, "Keep strings quoted instead of unquoting them")
	flag.BoolVar(&indent, "i", false, "Nicely indent any JSON output")
	flag.Parse()

	errored := false

	args := flag.Args()
	if len(args) == 0 {
		args = append(args, ":root")
	}

	scanner := bufio.NewScanner(os.Stdin)
	for scanner.Scan() {
		elements, err := elementsForAllPatterns(scanner.Text(), args)
		if err != nil {
			log.Println("Error:", err)
			return
		}

		for i, el := range elements {
			if i > 0 {
				if singleLine {
					fmt.Print("\t")
				} else {
					fmt.Print("\n")
				}
			}

			stringElement, isString := el.(string)
			if !quotedStrings && isString {
				fmt.Print(stringElement)
			} else {
				var out []byte
				if indent {
					out, err = json.MarshalIndent(el, "", "  ")
				} else {
					out, err = json.Marshal(el)
				}
				if err != nil {
					log.Printf("Error marshalling %v: %s\n", el, err)
					errored = true
				}

{'fraction_non_alphanumeric': 0.1391968465139197, 'fraction_numerical': 0.0007390983000739098, 'mean_word_length': 2.881453154875717, 'pattern_counts': {'":': 1, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '763157', 'n_tokens_mistral': 1517, 'n_tokens_neox': 1337, 'n_words': 440}