<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	
	"github.com/phil-mansfield/gotetra/render/halo"
	"github.com/phil-mansfield/gotetra/render/io"
)

var valMap = map[string]halo.Val {
	"scale":halo.Scale,
	"id":halo.ID,
    "descscale":halo.DescScale,
    "descid":halo.DescID,
    "numprog":halo.NumProg,
    "pid":halo.PID,
    "upid":halo.UPID,
    "descpid":halo.DescPID,
    "phantom":halo.Phantom,
    "sammvir":halo.SAMMVir,
    "mvir":halo.MVir,
    "rvir":halo.RVir,
    "rs":halo.Rs,
    "vrms":halo.Vrms,
    "mmp":halo.MMP,
    "scaleoflastmmp":halo.ScaleOfLastMMP,
    "vmax":halo.VMax,
    "x":halo.X,
    "y":halo.Y,
    "z":halo.Z,
    "vx":halo.Vx,
    "vy":halo.Vy,
    "vz":halo.Vz,
    "jx":halo.Jx,
    "jy":halo.Jy,
    "jz":halo.Jz,
    "spin":halo.Spin,
    "breadthfirstid":halo.BreadthFirstID,
    "depthfirstid":halo.DepthFirstID,
    "treerootid":halo.TreeRootID,
    "orighaloid":halo.OrigHaloID,
    "snapnum":halo.SnapNum,
    "nextcoprogenitordepthfirstid":halo.NextCoprogenitorDepthFirstID,
    "lastprogenitordepthfirstid":halo.LastProgenitorDepthFirstID,
    "rsklypin":halo.RsKylpin,
    "mvirall":halo.MVirAll,
    "m200b":halo.M200b,
    "m200c":halo.M200c,
    "m500c":halo.M500c,
    "m2500c":halo.M2500c,
    "xoff":halo.XOff,
    "voff":halo.Voff,
    "spinbullock":halo.SpinBullock,
    "btoa":halo.BToA,
    "ctoa":halo.CToA,
    "ax":halo.Ax,
    "ay":halo.Ay,
    "az":halo.Az,
    "btoa500c":halo.BToA500c,
    "ctoa500c":halo.CToA500c,
    "ax500c":halo.Ax500c,
    "ay500c":halo.Ay500c,
    "az500c":halo.Az500c,
    "tu":halo.TU,
    "macc":halo.MAcc,
    "mpeak":halo.MPeak,
    "vacc":halo.VAcc,
    "vpeak":halo.VPeak,
    "halfmassscale":halo.HalfmassScale,
    "accrateinst":halo.AccRateInst,
    "accrate100myr":halo.AccRate100Myr,
    "accratetdyn":halo.AccRateTdyn,
}

func main() {
	valFlags, err := parseCmd()
	if err != nil { log.Fatal(err.Error()) }
	if len(valFlags) == 0 {
		err = printStdin()
		if err != nil { log.Fatal(err.Error()) }
	}
	
	ids, snaps, inVals, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	vals, err := readVals(ids, snaps, valFlags)
	if err != nil { log.Fatal(err.Error()) }

	printVals(ids, snaps, inVals, vals)
}


func parseCmd() ([]halo.Val, error) {
	flag.Parse()
	args := flag.Args()
	vals := make([]halo.Val, len(args))
	var ok bool
	for i, arg := range args {
		vals[i], ok = valMap[strings.ToLower(arg)]
		if !ok {
			return nil, fmt.Errorf("Flag %d, %s, not recognized.", i+1, arg)
		}
	}
	return vals, nil
}

func printStdin() error {
	lines, err := stdinLines()
	if err != nil { return err }
	fmt.Println(strings.Join(lines, "\n"))
	return nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseStdin() (ids, snaps []int, inVals [][]float64, err error) {
	ids, snaps, inVals = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		tokens := strings.Split(line, " ")
<commit_msg>Fixed tokenization of gtet_append.<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path"
	"strconv"
	"strings"
	
	"github.com/phil-mansfield/gotetra/render/halo"
	"github.com/phil-mansfield/gotetra/render/io"
)

var valMap = map[string]halo.Val {
	"scale":halo.Scale,
	"id":halo.ID,
    "descscale":halo.DescScale,
    "descid":halo.DescID,
    "numprog":halo.NumProg,
    "pid":halo.PID,
    "upid":halo.UPID,
    "descpid":halo.DescPID,
    "phantom":halo.Phantom,
    "sammvir":halo.SAMMVir,
    "mvir":halo.MVir,
    "rvir":halo.RVir,
    "rs":halo.Rs,
    "vrms":halo.Vrms,
    "mmp":halo.MMP,
    "scaleoflastmmp":halo.ScaleOfLastMMP,
    "vmax":halo.VMax,
    "x":halo.X,
    "y":halo.Y,
    "z":halo.Z,
    "vx":halo.Vx,
    "vy":halo.Vy,
    "vz":halo.Vz,
    "jx":halo.Jx,
    "jy":halo.Jy,
    "jz":halo.Jz,
    "spin":halo.Spin,
    "breadthfirstid":halo.BreadthFirstID,
    "depthfirstid":halo.DepthFirstID,
    "treerootid":halo.TreeRootID,
    "orighaloid":halo.OrigHaloID,
    "snapnum":halo.SnapNum,
    "nextcoprogenitordepthfirstid":halo.NextCoprogenitorDepthFirstID,
    "lastprogenitordepthfirstid":halo.LastProgenitorDepthFirstID,
    "rsklypin":halo.RsKylpin,
    "mvirall":halo.MVirAll,
    "m200b":halo.M200b,
    "m200c":halo.M200c,
    "m500c":halo.M500c,
    "m2500c":halo.M2500c,
    "xoff":halo.XOff,
    "voff":halo.Voff,
    "spinbullock":halo.SpinBullock,
    "btoa":halo.BToA,
    "ctoa":halo.CToA,
    "ax":halo.Ax,
    "ay":halo.Ay,
    "az":halo.Az,
    "btoa500c":halo.BToA500c,
    "ctoa500c":halo.CToA500c,
    "ax500c":halo.Ax500c,
    "ay500c":halo.Ay500c,
    "az500c":halo.Az500c,
    "tu":halo.TU,
    "macc":halo.MAcc,
    "mpeak":halo.MPeak,
    "vacc":halo.VAcc,
    "vpeak":halo.VPeak,
    "halfmassscale":halo.HalfmassScale,
    "accrateinst":halo.AccRateInst,
    "accrate100myr":halo.AccRate100Myr,
    "accratetdyn":halo.AccRateTdyn,
}

func main() {
	valFlags, err := parseCmd()
	if err != nil { log.Fatal(err.Error()) }
	if len(valFlags) == 0 {
		err = printStdin()
		if err != nil { log.Fatal(err.Error()) }
	}
	
	ids, snaps, inVals, err := parseStdin()
	if err != nil { log.Fatal(err.Error()) }
	vals, err := readVals(ids, snaps, valFlags)
	if err != nil { log.Fatal(err.Error()) }

	printVals(ids, snaps, inVals, vals)
}


func parseCmd() ([]halo.Val, error) {
	flag.Parse()
	args := flag.Args()
	vals := make([]halo.Val, len(args))
	var ok bool
	for i, arg := range args {
		vals[i], ok = valMap[strings.ToLower(arg)]
		if !ok {
			return nil, fmt.Errorf("Flag %d, %s, not recognized.", i+1, arg)
		}
	}
	return vals, nil
}

func printStdin() error {
	lines, err := stdinLines()
	if err != nil { return err }
	fmt.Println(strings.Join(lines, "\n"))
	return nil
}

func stdinLines() ([]string, error) {
	bs, err := ioutil.ReadAll(os.Stdin)
	if err != nil {
		return nil, fmt.Errorf(
			"Error reading stdin: %s.", err.Error(),
		)
	}

	text := string(bs)
	return strings.Split(text, "\n"), nil
}

func parseStdin() (ids, snaps []int, inVals [][]float64, err error) {
	ids, snaps, inVals = []int{}, []int{}, [][]float64{}
	lines, err := stdinLines()
	if err != nil { return nil, nil, nil, err }
	for i, line := range lines {
		rawTokens := strings.Split(line, " ")
		tokens := make([]string, 0, len(rawTokens))
		for _, tok := range rawTokens {
			if len(tok) != 0 { tokens = append(tokens, tok) }
		}

{'fraction_non_alphanumeric': 0.19036108173440097, 'fraction_numerical': 0.02084907085662487, 'mean_word_length': 3.5749827228749136, 'pattern_counts': {'":': 124, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3590177', 'n_tokens_mistral': 3166, 'n_tokens_neox': 2964, 'n_words': 467}