<commit_before>//  Copyright (c) 2013 Couchbase, Inc.

package main

import "flag"
import "fmt"
import "log"
import "os"
import "time"
import "io/ioutil"
import _ "net/http/pprof"
import "net/http"
import "runtime/pprof"

import "github.com/prataprc/goparsec"
import "github.com/prataprc/monster"
import "github.com/prataprc/monster/common"

var options struct {
	bagdir  string
	outfile string
	nonterm string
	memprof string
	seed    int
	count   int
	par     int
	help    bool
	debug   bool
}

func argParse() (string, *os.File) {
	seed := time.Now().UTC().Second()
	flag.StringVar(&options.bagdir, "bagdir", "",
		"directory path containing bags")
	flag.StringVar(&options.nonterm, "nonterm", "s",
		"evaluate the non-terminal")
	flag.StringVar(&options.memprof, "memprof", "",
		"dump mem-profile to file")
	flag.IntVar(&options.seed, "seed", seed,
		"seed value")
	flag.IntVar(&options.count, "count", 1,
		"generate count number of combinations")
	flag.IntVar(&options.par, "par", 1,
		"number of parallel routines to use to generate")
	flag.StringVar(&options.outfile, "o", "-",
		"specify an output file")
	flag.Parse()

	prodfile := flag.Args()[0]
	if prodfile == "" || options.help {
		usage()
		os.Exit(1)
	}

	var err error
	outfd := os.Stdout
	if options.outfile != "-" && options.outfile != "" {
		outfd, err = os.Create(options.outfile)
		if err != nil {
			log.Fatal(err)
		}
	}
	return prodfile, outfd
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage : %s [OPTIONS] <production-file> \n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	prodfile, outfd := argParse()

	// start pprof
	go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }()

	// read production-file
	text, err := ioutil.ReadFile(prodfile)
	if err != nil {
		log.Fatal(err)
	}

	// spawn generators
	outch := make(chan []byte, 1000)
	for i := 0; i < options.par; i++ {
		go generate(text, options.count, prodfile, outch)
	}

	// gather generated values
	till := options.count * options.par
	for till > 0 {
		outtext := <-outch
		outtext = append(outtext, '\n')
		if _, err := outfd.Write([]byte(outtext)); err != nil {
			log.Fatal(err)
		}
		till--
	}
	fmt.Println("Completed !!")
	if takeMEMProfile(options.memprof) {
		fmt.Printf("dumped mem-profile to %v\n", options.memprof)
	}
}

func generate(text []byte, count int, prodfile string, outch chan<- []byte) {
	// compile
	root := compile(parsec.NewScanner(text)).(common.Scope)
	seed, bagdir, prodfile := uint64(options.seed), options.bagdir, prodfile
	scope := monster.BuildContext(root, seed, bagdir, prodfile)
	nterms := scope["_nonterminals"].(common.NTForms)
	for i := 0; i < count; i++ {
		scope = scope.RebuildContext()
		val := evaluate("root", scope, nterms[options.nonterm])
		outch <- []byte(val.(string))
	}
}

func compile(s parsec.Scanner) parsec.ParsecNode {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("%v at %v", r, s.GetCursor())
		}
	}()
	root, _ := monster.Y(s)
	return root
}

func evaluate(name string, scope common.Scope, forms []*common.Form) interface{} {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("%v", r)
		}
	}()
	return monster.EvalForms(name, scope, forms)
}

func takeMEMProfile(filename string) bool {
	if filename == "" {
		return false
	}
	fd, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
		return false
	}
	pprof.WriteHeapProfile(fd)
	defer fd.Close()
	return true
}
<commit_msg>verify the validity of json generated from .prod files<commit_after>//  Copyright (c) 2013 Couchbase, Inc.

package main

import "encoding/json"
import "flag"
import "fmt"
import "log"
import "os"
import "time"
import "io/ioutil"
import _ "net/http/pprof"
import "net/http"
import "runtime/pprof"

import "github.com/prataprc/goparsec"
import "github.com/prataprc/monster"
import "github.com/prataprc/monster/common"

var options struct {
	bagdir  string
	outfile string
	nonterm string
	memprof string
	seed    int
	count   int
	par     int
	help    bool
	debug   bool
}

func argParse() (string, *os.File) {
	seed := time.Now().UTC().Second()
	flag.StringVar(&options.bagdir, "bagdir", "",
		"directory path containing bags")
	flag.StringVar(&options.nonterm, "nonterm", "s",
		"evaluate the non-terminal")
	flag.StringVar(&options.memprof, "memprof", "",
		"dump mem-profile to file")
	flag.IntVar(&options.seed, "seed", seed,
		"seed value")
	flag.IntVar(&options.count, "count", 1,
		"generate count number of combinations")
	flag.IntVar(&options.par, "par", 1,
		"number of parallel routines to use to generate")
	flag.StringVar(&options.outfile, "o", "-",
		"specify an output file")
	flag.Parse()

	prodfile := flag.Args()[0]
	if prodfile == "" || options.help {
		usage()
		os.Exit(1)
	}

	var err error
	outfd := os.Stdout
	if options.outfile != "-" && options.outfile != "" {
		outfd, err = os.Create(options.outfile)
		if err != nil {
			log.Fatal(err)
		}
	}
	return prodfile, outfd
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage : %s [OPTIONS] <production-file> \n", os.Args[0])
	flag.PrintDefaults()
}

func main() {
	prodfile, outfd := argParse()

	// start pprof
	go func() { log.Println(http.ListenAndServe("localhost:6060", nil)) }()

	// read production-file
	text, err := ioutil.ReadFile(prodfile)
	if err != nil {
		log.Fatal(err)
	}

	// spawn generators
	outch := make(chan []byte, 1000)
	for i := 0; i < options.par; i++ {
		go generate(text, options.count, prodfile, outch)
	}

	// gather generated values
	till := options.count * options.par
	for till > 0 {
		outtext := <-outch
		outtext = append(outtext, '\n')
		if _, err := outfd.Write([]byte(outtext)); err != nil {
			log.Fatal(err)
		}
		till--
	}
	fmt.Println("Completed !!")
	if takeMEMProfile(options.memprof) {
		fmt.Printf("dumped mem-profile to %v\n", options.memprof)
	}
}

func generate(text []byte, count int, prodfile string, outch chan<- []byte) {
	// compile
	root := compile(parsec.NewScanner(text)).(common.Scope)
	seed, bagdir, prodfile := uint64(options.seed), options.bagdir, prodfile
	scope := monster.BuildContext(root, seed, bagdir, prodfile)
	nterms := scope["_nonterminals"].(common.NTForms)
	for i := 0; i < count; i++ {
		scope = scope.RebuildContext()
		val := evaluate("root", scope, nterms[options.nonterm])

		// Verify the validity of JSON generated
		var value map[string]interface{}
		err := json.Unmarshal([]byte(val.(string)), &value)
		if err != nil {
			log.Printf("Invalid JSON %v\n", err)
		} else {
			outch <- []byte(val.(string))
		}
	}
}

func compile(s parsec.Scanner) parsec.ParsecNode {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("%v at %v", r, s.GetCursor())
		}
	}()
	root, _ := monster.Y(s)
	return root
}

func evaluate(name string, scope common.Scope, forms []*common.Form) interface{} {
	defer func() {
		if r := recover(); r != nil {
			log.Printf("%v", r)
		}
	}()
	return monster.EvalForms(name, scope, forms)
}

func takeMEMProfile(filename string) bool {
	if filename == "" {
		return false
	}
	fd, err := os.Create(filename)
	if err != nil {
		log.Fatal(err)
		return false
	}
	pprof.WriteHeapProfile(fd)
	defer fd.Close()
	return true
}

{'fraction_non_alphanumeric': 0.16612880594920723, 'fraction_numerical': 0.00617370562649081, 'mean_word_length': 3.852280462899932, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '872996', 'n_tokens_mistral': 2764, 'n_tokens_neox': 2612, 'n_words': 654}