<commit_before>package main

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"io/ioutil"
	"os"

	"github.com/polydawn/refmt/cbor"
	"github.com/polydawn/refmt/pretty"
	"github.com/polydawn/refmt/shared"
)

func main() {
	in, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		panic(err)
	}
	bs := make([]byte, len(in)/2)
	n, err := hex.Decode(bs, in)
	if err != nil {
		panic(err)
	}
	if n != len(in)/2 {
		panic(fmt.Errorf("hex len mismatch: %d chars became %d bytes", len(in), n))
	}

	pump := shared.TokenPump{
		cbor.NewDecoder(bytes.NewBuffer(bs)),
		pretty.NewEncoder(os.Stdout),
	}
	if err := pump.Run(); err != nil {
		panic(err)
	}
}<commit_msg>cmd: start developing nice cli.<commit_after>package main

import (
	"bytes"
	"encoding/hex"
	"fmt"
	"io"
	"io/ioutil"
	"os"

	"github.com/urfave/cli"

	"github.com/polydawn/refmt/cbor"
	"github.com/polydawn/refmt/pretty"
	"github.com/polydawn/refmt/shared"
)

func main() {
	os.Exit(Main(os.Args, os.Stdin, os.Stdout, os.Stderr))
}

func Main(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
	app := cli.NewApp()
	app.Name = "refmt"
	app.Authors = []cli.Author{
		cli.Author{Name: "<NAME>", Email: "<EMAIL>"},
	}
	app.Commands = []cli.Command{
		cli.Command{
			Category: "convert",
			Name:     "hex=cbor=pretty",
			Usage:    "read cbor in hex, then pretty print it",
			Action: func(c *cli.Context) error {
				fmt.Fprintf(stderr, "hello\n")
				in, err := ioutil.ReadAll(os.Stdin)
				if err != nil {
					return err
				}
				bs := make([]byte, len(in)/2)
				n, err := hex.Decode(bs, in)
				if err != nil {
					return err
				}
				if n != len(in)/2 {
					return fmt.Errorf("hex len mismatch: %d chars became %d bytes", len(in), n)
				}

				pump := shared.TokenPump{
					cbor.NewDecoder(bytes.NewBuffer(bs)),
					pretty.NewEncoder(os.Stdout),
				}
				return pump.Run()
			},
		},
	}
	app.Writer = stdout
	app.ErrWriter = stderr
	err := app.Run(args)
	if err != nil {
		fmt.Fprintf(stderr, "error: %s\n", err)
		return 1
	}
	return 0
}
{'fraction_non_alphanumeric': 0.17992047713717693, 'fraction_numerical': 0.002982107355864811, 'mean_word_length': 3.1505154639175257, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2048610', 'n_tokens_mistral': 934, 'n_tokens_neox': 836, 'n_words': 175}