<commit_before>package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/rwcarlsen/cyan/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid"
	"github.com/rwcarlsen/cyan/nuc"
	"github.com/rwcarlsen/cyan/post"
	"github.com/rwcarlsen/cyan/query"
	_ "github.com/rwcarlsen/go-sqlite/sqlite3"
)

var (
	custom    = flag.String("custom", "", "path to custom sql query spec file")
	showquery = flag.Bool("query", false, "show query SQL for a subcommand instead of executing it")
	dbname    = flag.String("db", "", "cyclus sqlite database to query")
	simidstr  = flag.String("simid", "", "simulation id in hex (empty string defaults to first sim id in database")
)

var simid []byte

var command string

var db *sql.DB

var cmds = NewCmdSet()

// map[cmdname]sqltext
var customSql = map[string]string{}

func plot(data *bytes.Buffer, style string, xlabel, ylabel, title string) {
	s := ""
	s += `set xlabel '{{.Xlabel}}';`
	s += `set ylabel '{{.Ylabel}}';`
	s += `plot '-' every ::2 using 1:2 with {{.Style}} title '{{.Title}}';`
	s += `pause -1`

	tmpl := template.Must(template.New("gnuplot").Parse(s))
	var buf bytes.Buffer
	config := struct{ Style, Xlabel, Ylabel, Title string }{style, xlabel, ylabel, title}
	err := tmpl.Execute(&buf, config)
	fatalif(err)

	cmd := exec.Command("gnuplot", "-p", "-e", buf.String())
	cmd.Stdin = data
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	fatalif(cmd.Run())
}

func init() {
	cmds.RegisterDiv("General")
	cmds.Register("sims", "list all simulations in the database", doSims)
	cmds.Register("infile", "show the simulation's input file", doInfile)
	cmds.Register("version", "show simulation's cyclus version info", doVersion)
	cmds.Register("post", "post process the database", doPost)
	cmds.Register("table", "show the contents of a specific table", doTable)
	cmds.Register("ts", "investigate time-series data tables", doTimeSeries)
	cmds.RegisterDiv("Agents")
	cmds.Register("agents", "list all agents in the simulation", doAgents)
	cmds.Register("protos", "list all prototypes in the simulation", doProtos)
	cmds.Register("deployed", "time series total active deployments by prototype", doDeployed)
	cmds.Register("built", "time series of new builds by prototype", doBuilt)
	cmds.Register("decom", "time series of a decommissionings by prototype", doDecom)
	cmds.RegisterDiv("Flow")
	cmds.Register("commods", "show commodity transaction counts and quantities", doCommods)
	cmds.Register("flow", "time series of material transacted between agents", doFlow)
	cmds.Register("flowgraph", "generate a graphviz dot script of flows between agents", doFlowGraph)
	cmds.Register("trans", "time series of transaction quantity over time", doTrans)
	cmds.RegisterDiv("Other")
	cmds.Register("inv", "time series of inventory by prototype", doInv)
	cmds.Register("power", "time series of power produced", doPower)
	cmds.Register("energy", "thermal energy (J) generated between 2 timesteps", doEnergy)
	cmds.Register("created", "material created by agents between 2 timesteps", doCreated)
}

func initdb() {
	if *showquery {
		// don't need a database for printing queries
		return
	} else if *dbname == "" {
		log.Fatal("must specify database with -db flag")
	}

	var err error
	db, err = sql.Open("sqlite3", *dbname)
	fatalif(err)

	if *simidstr == "" {
		ids, err := query.SimIds(db)
		fatalif(err)
		simid = ids[0]
	} else {
		simid = uuid.Parse(*simidstr)
		if simid == nil {
			log.Fatalf("invalid simid '%s'", *simidstr)
		}
	}

	post.Process(db)
}

func main() {
	log.SetFlags(0)
	flag.CommandLine.Usage = func() {
		fmt.Println("Usage: cyan [-db <cyclus-db>] [flags...] <subcommand> [flags...] [args...]")
		fmt.Println("Computes metrics for cyclus simulation data in a sqlite database.")
		fmt.Println("\nOptions:")
		flag.CommandLine.PrintDefaults()
		fmt.Println("\nSub-commands:")
		tw := tabwriter.NewWriter(os.Stdout, 2, 2, 2, ' ', 0)
		for i := range cmds.Names {
			if cmds.IsDiv(i) {
				fmt.Fprintf(tw, "\n\t[%v]\n", cmds.Names[i])
			} else {
				fmt.Fprintf(tw, "\t\t%v\t%v\n", cmds.Names[i], cmds.Helps[i])
			}
		}
		tw.Flush()
	}
	flag.Parse()

	if flag.NArg() < 1 {
		fmt.Println("Usage: cyan -db <cyclus-db> [flags...] <command> [flags...] [args...]")
		fmt.Println("Computes metrics for cyclus simulation data in a sqlite database.")
		flag.PrintDefaults()
		fmt.Println("\nSub-commands:")
		for i := range cmds.Names {
			fmt.Printf("    %v: %v\n", cmds.Names[i], cmds.Helps[i])
		}
		return
	}

	if *custom != "" {
		data, err := ioutil.ReadFile(*custom)
		fatalif(err)
		fatalif(json.Unmarshal(data, &customSql))
	}

	// run command
	cmds.Execute(flag.Args())
}

func doCustom(w io.Writer, cmd string, args ...interface{}) {
	s, ok := customSql[cmd]
	if !ok {
		log.Fatalf("Invalid command/query %v", cmd)
	} else if *showquery {
		fmt.Fprint(w, s)
		return
	}

	rows, err := db.Query(s, args...)
	fatalif(err)

	tw := tabwriter.NewWriter(w, 4, 4, 1, ' ', 0)
	cols, err := rows.Columns()
	fatalif(err)

	simidcol := -1
	for i, c := range cols {
		if strings.Contains(strings.ToLower(c), "simid") {
			simidcol = i
		}
		_, err := tw.Write([]byte(c + "\t"))
		fatalif(err)
	}
	_, err = tw.Write([]byte("\n"))
	fatalif(err)

	vs := make([]interface{}, len(cols))
	vals := make([]*sql.NullString, len(cols))
	for i := range vals {
		vals[i] = &sql.NullString{}
		vs[i] = vals[i]
	}

	for rows.Next() {
		for i := range vals {
			vals[i].Valid = false
		}

		err := rows.Scan(vs...)
		fatalif(err)

		for i, v := range vals {
			if v.Valid {
				s = v.String
				if i == simidcol {
					s = uuid.UUID(v.String).String()
				}
				tw.Write([]byte(s + "\t"))
			} else {
				tw.Write([]byte("NULL\t"))
			}
		}

		_, err = tw.Write([]byte("\n"))
		fatalif(err)
	}
	fatalif(rows.Err())
	fatalif(tw.Flush())
}

func doSims(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
	s := `
SELECT i.SimId AS SimId,Duration,Handle,Decay
FROM Info As i
JOIN DecayMode AS d ON i.SimId=d.SimId 
`
	customSql[cmd] = s
	doCustom(os.Stdout, cmd)
}

func doVersion(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	av := fs.Bool("agent", false, "show simulation's agent versions instead")
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
	s := `
SELECT
i.CyclusVersionDescribe AS Cyclus,
i.SqliteVersion As SQLite,
i.Hdf5Version AS HDF5,
i.BoostVersion AS Boost,
i.LibXML2Version AS libxml2,
i.CoinCBCVersion AS CoinCBC,
x.LibXMLPlusPlusVersion AS 'libxml++'
FROM info AS i
JOIN xmlppinfo AS x ON i.simid=x.simid
WHERE i.simid=?
`
	if *av {
		s = `SELECT Spec AS Archetype,Version FROM AgentVersions WHERE simid=?`
	}
	customSql[cmd] = s
	doCustom(os.Stdout, cmd, simid)
}

func doPost(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
}

func doInfile(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
	s := `
SELECT data FROM inputfiles WHERE simid=?;
`
	var buf bytes.Buffer
	customSql[cmd] = s
	doCustom(&buf, cmd, simid)
	data := buf.String()
	data = strings.Replace(data, "Data", "", 1)
	data = strings.TrimLeft(data, "\r\n\t ")
	fmt.Print(data)
}

func doAgents(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	proto := fs.String("proto", "", "filter by prototype (default is all prototypes)")
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()

	iargs := []interface{}{simid}
	s := `
SELECT AgentId,Kind,Prototype,ParentId,EnterTime,ExitTime,Lifetime
FROM Agents
WHERE SimId = ?
`
	if *proto != "" {
		s += ` AND Prototype = ?`
		iargs = append(iargs, *proto)
	}
	customSql[cmd] = s
	doCustom(os.Stdout, cmd, iargs...)
}

func doTable(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v [table-name]", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		log.Printf("If no table name is given, prints a list of tables.")
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()

	if fs.NArg() > 0 {
		s := "SELECT * FROM " + fs.Arg(0) + " WHERE SimId = ?"
		customSql[cmd] = s
		doCustom(os.Stdout, cmd, simid)
	} else {
		s := "SELECT name FROM sqlite_master WHERE type='table';"
		customSql[cmd] = s
		doCustom(os.Stdout, cmd)
	}
}

func doTimeSeries(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	proto := fs.String("proto", "", "filter by prototype (default is all prototypes)")<commit_msg>add ages subcommand for agent age distributions<commit_after>package main

import (
	"bytes"
	"database/sql"
	"encoding/json"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"strconv"
	"strings"
	"text/tabwriter"
	"text/template"

	"github.com/rwcarlsen/cyan/Godeps/_workspace/src/code.google.com/p/go-uuid/uuid"
	"github.com/rwcarlsen/cyan/nuc"
	"github.com/rwcarlsen/cyan/post"
	"github.com/rwcarlsen/cyan/query"
	_ "github.com/rwcarlsen/go-sqlite/sqlite3"
)

var (
	custom    = flag.String("custom", "", "path to custom sql query spec file")
	showquery = flag.Bool("query", false, "show query SQL for a subcommand instead of executing it")
	dbname    = flag.String("db", "", "cyclus sqlite database to query")
	simidstr  = flag.String("simid", "", "simulation id in hex (empty string defaults to first sim id in database")
)

var simid []byte

var command string

var db *sql.DB

var cmds = NewCmdSet()

// map[cmdname]sqltext
var customSql = map[string]string{}

func plot(data *bytes.Buffer, style string, xlabel, ylabel, title string) {
	s := ""
	s += `set xlabel '{{.Xlabel}}';`
	s += `set ylabel '{{.Ylabel}}';`
	s += `plot '-' every fd00:c2b6:b24b:be67:2827:688d:e6a1:6a3b using 1:2 with {{.Style}} title '{{.Title}}';`
	s += `pause -1`

	tmpl := template.Must(template.New("gnuplot").Parse(s))
	var buf bytes.Buffer
	config := struct{ Style, Xlabel, Ylabel, Title string }{style, xlabel, ylabel, title}
	err := tmpl.Execute(&buf, config)
	fatalif(err)

	cmd := exec.Command("gnuplot", "-p", "-e", buf.String())
	cmd.Stdin = data
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr
	fatalif(cmd.Run())
}

func init() {
	cmds.RegisterDiv("General")
	cmds.Register("sims", "list all simulations in the database", doSims)
	cmds.Register("infile", "show the simulation's input file", doInfile)
	cmds.Register("version", "show simulation's cyclus version info", doVersion)
	cmds.Register("post", "post process the database", doPost)
	cmds.Register("table", "show the contents of a specific table", doTable)
	cmds.Register("ts", "investigate time-series data tables", doTimeSeries)
	cmds.RegisterDiv("Agents")
	cmds.Register("agents", "list all agents in the simulation", doAgents)
	cmds.Register("protos", "list all prototypes in the simulation", doProtos)
	cmds.Register("deployed", "time series total active deployments by prototype", doDeployed)
	cmds.Register("built", "time series of new builds by prototype", doBuilt)
	cmds.Register("decom", "time series of a decommissionings by prototype", doDecom)
	cmds.Register("ages", "list ages of agents at a particular time step", doAges)
	cmds.RegisterDiv("Flow")
	cmds.Register("commods", "show commodity transaction counts and quantities", doCommods)
	cmds.Register("flow", "time series of material transacted between agents", doFlow)
	cmds.Register("flowgraph", "generate a graphviz dot script of flows between agents", doFlowGraph)
	cmds.Register("trans", "time series of transaction quantity over time", doTrans)
	cmds.RegisterDiv("Other")
	cmds.Register("inv", "time series of inventory by prototype", doInv)
	cmds.Register("power", "time series of power produced", doPower)
	cmds.Register("energy", "thermal energy (J) generated between 2 timesteps", doEnergy)
	cmds.Register("created", "material created by agents between 2 timesteps", doCreated)
}

func initdb() {
	if *showquery {
		// don't need a database for printing queries
		return
	} else if *dbname == "" {
		log.Fatal("must specify database with -db flag")
	}

	var err error
	db, err = sql.Open("sqlite3", *dbname)
	fatalif(err)

	if *simidstr == "" {
		ids, err := query.SimIds(db)
		fatalif(err)
		simid = ids[0]
	} else {
		simid = uuid.Parse(*simidstr)
		if simid == nil {
			log.Fatalf("invalid simid '%s'", *simidstr)
		}
	}

	post.Process(db)
}

func main() {
	log.SetFlags(0)
	flag.CommandLine.Usage = func() {
		fmt.Println("Usage: cyan [-db <cyclus-db>] [flags...] <subcommand> [flags...] [args...]")
		fmt.Println("Computes metrics for cyclus simulation data in a sqlite database.")
		fmt.Println("\nOptions:")
		flag.CommandLine.PrintDefaults()
		fmt.Println("\nSub-commands:")
		tw := tabwriter.NewWriter(os.Stdout, 2, 2, 2, ' ', 0)
		for i := range cmds.Names {
			if cmds.IsDiv(i) {
				fmt.Fprintf(tw, "\n\t[%v]\n", cmds.Names[i])
			} else {
				fmt.Fprintf(tw, "\t\t%v\t%v\n", cmds.Names[i], cmds.Helps[i])
			}
		}
		tw.Flush()
	}
	flag.Parse()

	if flag.NArg() < 1 {
		fmt.Println("Usage: cyan -db <cyclus-db> [flags...] <command> [flags...] [args...]")
		fmt.Println("Computes metrics for cyclus simulation data in a sqlite database.")
		flag.PrintDefaults()
		fmt.Println("\nSub-commands:")
		for i := range cmds.Names {
			fmt.Printf("    %v: %v\n", cmds.Names[i], cmds.Helps[i])
		}
		return
	}

	if *custom != "" {
		data, err := ioutil.ReadFile(*custom)
		fatalif(err)
		fatalif(json.Unmarshal(data, &customSql))
	}

	// run command
	cmds.Execute(flag.Args())
}

func doCustom(w io.Writer, cmd string, args ...interface{}) {
	s, ok := customSql[cmd]
	if !ok {
		log.Fatalf("Invalid command/query %v", cmd)
	} else if *showquery {
		fmt.Fprint(w, s)
		return
	}

	rows, err := db.Query(s, args...)
	fatalif(err)

	tw := tabwriter.NewWriter(w, 4, 4, 1, ' ', 0)
	cols, err := rows.Columns()
	fatalif(err)

	simidcol := -1
	for i, c := range cols {
		if strings.Contains(strings.ToLower(c), "simid") {
			simidcol = i
		}
		_, err := tw.Write([]byte(c + "\t"))
		fatalif(err)
	}
	_, err = tw.Write([]byte("\n"))
	fatalif(err)

	vs := make([]interface{}, len(cols))
	vals := make([]*sql.NullString, len(cols))
	for i := range vals {
		vals[i] = &sql.NullString{}
		vs[i] = vals[i]
	}

	for rows.Next() {
		for i := range vals {
			vals[i].Valid = false
		}

		err := rows.Scan(vs...)
		fatalif(err)

		for i, v := range vals {
			if v.Valid {
				s = v.String
				if i == simidcol {
					s = uuid.UUID(v.String).String()
				}
				tw.Write([]byte(s + "\t"))
			} else {
				tw.Write([]byte("NULL\t"))
			}
		}

		_, err = tw.Write([]byte("\n"))
		fatalif(err)
	}
	fatalif(rows.Err())
	fatalif(tw.Flush())
}

func doSims(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
	s := `
SELECT i.SimId AS SimId,Duration,Handle,Decay
FROM Info As i
JOIN DecayMode AS d ON i.SimId=d.SimId 
`
	customSql[cmd] = s
	doCustom(os.Stdout, cmd)
}

func doVersion(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	av := fs.Bool("agent", false, "show simulation's agent versions instead")
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
	s := `
SELECT
i.CyclusVersionDescribe AS Cyclus,
i.SqliteVersion As SQLite,
i.Hdf5Version AS HDF5,
i.BoostVersion AS Boost,
i.LibXML2Version AS libxml2,
i.CoinCBCVersion AS CoinCBC,
x.LibXMLPlusPlusVersion AS 'libxml++'
FROM info AS i
JOIN xmlppinfo AS x ON i.simid=x.simid
WHERE i.simid=?
`
	if *av {
		s = `SELECT Spec AS Archetype,Version FROM AgentVersions WHERE simid=?`
	}
	customSql[cmd] = s
	doCustom(os.Stdout, cmd, simid)
}

func doPost(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
}

func doInfile(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()
	s := `
SELECT data FROM inputfiles WHERE simid=?;
`
	var buf bytes.Buffer
	customSql[cmd] = s
	doCustom(&buf, cmd, simid)
	data := buf.String()
	data = strings.Replace(data, "Data", "", 1)
	data = strings.TrimLeft(data, "\r\n\t ")
	fmt.Print(data)
}

func doAgents(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	proto := fs.String("proto", "", "filter by prototype (default is all prototypes)")
	fs.Usage = func() {
		log.Printf("Usage: %v", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()

	iargs := []interface{}{simid}
	s := `
SELECT AgentId,Kind,Prototype,ParentId,EnterTime,ExitTime,Lifetime
FROM Agents
WHERE SimId = ?
`
	if *proto != "" {
		s += `AND Prototype = ?`
		iargs = append(iargs, *proto)
	}
	customSql[cmd] = s
	doCustom(os.Stdout, cmd, iargs...)
}

func doAges(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	proto := fs.String("proto", "", "filter by prototype (default is all prototypes)")
	fs.Usage = func() {
		log.Printf("Usage: %v [time-step]", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()

	t, err := strconv.Atoi(fs.Arg(0))
	if err != nil {
		log.Fatalf("invalid time step '%v')", fs.Arg(0))
	}

	iargs := []interface{}{t, simid, t, t}
	s := `
SELECT ? - a.entertime AS Age FROM Agents as a
WHERE a.simid=?
AND a.entertime <= ?
AND (a.exittime >= ? OR a.exittime ISNULL)
`
	if *proto != "" {
		s += `AND Prototype = ?`
		iargs = append(iargs, *proto)
	}

	customSql[cmd] = s
	doCustom(os.Stdout, cmd, iargs...)
}

func doTable(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	fs.Usage = func() {
		log.Printf("Usage: %v [table-name]", cmd)
		log.Printf("%v\n", cmds.Help(cmd))
		log.Printf("If no table name is given, prints a list of tables.")
		fs.PrintDefaults()
	}
	fs.Parse(args)
	initdb()

	if fs.NArg() > 0 {
		s := "SELECT * FROM " + fs.Arg(0) + " WHERE SimId = ?"
		customSql[cmd] = s
		doCustom(os.Stdout, cmd, simid)
	} else {
		s := "SELECT name FROM sqlite_master WHERE type='table';"
		customSql[cmd] = s
		doCustom(os.Stdout, cmd)
	}
}

func doTimeSeries(cmd string, args []string) {
	fs := flag.NewFlagSet(cmd, flag.ExitOnError)
	proto := fs.String("proto", "", "filter by prototype (default is all prototypes)")
{'fraction_non_alphanumeric': 0.17525069564760853, 'fraction_numerical': 0.0038851262666036645, 'mean_word_length': 4.032496697490092, 'pattern_counts': {'":': 0, '<': 14, '<?xml version=': 0, '>': 14, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 6}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1305072', 'n_tokens_mistral': 7674, 'n_tokens_neox': 7260, 'n_words': 1789}