<commit_before>// Package json generates methods for encoding/decoding types to/from JSON.
//
// When used correctly, these methods can easily give a ~200-300% performance
// increase when serializing objects to JSON while also reducing memory usage
// by ~95-99%. For taking advantage of these gains, you must use
// gnd.la/mux/serialize or Context.WriteJson to encode to JSON, since
// json.Marshal won't use these methods correctly and might even have worse
// performance when these methods are implemented.
//
// This is a small benchmark comparing the performance of these JSON encoding
// methods. JSONDirect uses WriteJSON(), JSONSerialize uses
// gnd.la/mux/serialize (which adds some overhead because it also sets the
// Content-Length and Content-Encoding headers and thus must encode into an
// intermediate buffer first), while JSON uses json.Marshal(). All three
// benchmarks write the result to ioutil.Discard.
//
//  BenchmarkJSONDirect	    1000000 1248 ns/op	117.73 MB/s 16 B/op	2 allocs/op
//  BenchmarkJSONSerialize  1000000 1587 ns/op	92.62 MB/s  16 B/op	2 allocs/op
//  BenchmarkJSON	    500000  4583 ns/op	32.07 MB/s  620 B/op	4 allocs/op
//
// Code geerated by this package respects json related struct tags and also
// supports encoding time.Time structs directly (they're encoded as a UTC unix time).
//
// The recommended way use to generate JSON methods for a given package is
// using the gondola command rather than using this package directly.
package json

import (
	"bytes"
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"gnd.la/gen/genutil"
	"gnd.la/log"
	gtypes "gnd.la/types"
	"path/filepath"
	"regexp"
	"strings"
)

type Method struct {
	Key       string
	Name      string
	OmitEmpty bool
}

// Options specify the options used when generating JSON related
// methods.
type Options struct {
	// Wheter to generate a MarshalJSON method. This is false by default
	// because in most cases will result in lower performance when using
	// json.Marshal, since the encoder from encoding/json will revalidate
	// the returned JSON, resulting in a performance loss. Turn this on
	// only if you're using the Methods feature (otherwise you'll get
	// different results when serializing with json.Marshal).
	MarshalJSON bool
	// Methods indicates struct methods which should be included in the JSON
	// output. The key in the map is the type name in the package (e.g.
	// MyStruct not mypackage.MyStruct).
	Methods map[string][]*Method
	// If not nil, only types matching this regexp will be included.
	Include *regexp.Regexp
	// If not nil, types matching this regexp will be excluded.
	Exclude *regexp.Regexp
}

// Gen generates a WriteJSON method and, optionally, MarshalJSON for every
// exported type in the given package. The package might be either an
// absolute path or an import path.
func Gen(pkgName string, opts *Options) error {
	pkg, err := genutil.NewPackage(pkgName)
	if err != nil {
		return err
	}
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("package %s\n\n", pkg.Name()))
	buf.WriteString(genutil.AutogenString())
	buf.WriteString("\nimport (\n")
	imports := []string{"bytes", "io", "runtime", "strconv", "unicode/utf8"}
	for _, v := range imports {
		buf.WriteString(fmt.Sprintf("%q\n", v))
	}
	buf.WriteString(")\n")
	buf.WriteString("var _ = strconv.FormatBool\n")
	scope := pkg.Scope()
	var methods bytes.Buffer
	prefix := pkg.Name() + "."
	for _, v := range scope.Names() {
		methods.Reset()
		obj := scope.Lookup(v)
		if !obj.IsExported() {
			continue
		}
		if named, ok := obj.Type().(*types.Named); ok && strings.HasPrefix(named.String(), prefix) && isIncluded(named.String()[len(prefix):], opts) {
			if err := jsonMarshal(obj, named, opts, &methods); err != nil {
				log.Warningf("Skipping type %s: %s", obj.Name(), err)
				continue
			}
			buf.WriteString(methods.String())
		}
	}
	buf.WriteString(encode_go)
	buf.WriteString(buffer_go)
	out := filepath.Join(pkg.Dir(), "gen_json.go")
	log.Debugf("Writing autogenerated JSON methods to %s", out)
	return genutil.WriteAutogen(out, buf.Bytes())
}

func isIncluded(name string, opts *Options) bool {
	if opts != nil {
		if opts.Exclude != nil && opts.Exclude.MatchString(name) {
			return false
		}
		if opts.Include != nil {
			return opts.Include.MatchString(name)
		}
	}
	return true
}

func jsonMarshal(obj types.Object, typ types.Type, opts *Options, buf *bytes.Buffer) error {
	tname := obj.Name()
	if _, ok := typ.(*types.Struct); ok {
		tname = "*" + tname
	}
	if opts != nil && opts.MarshalJSON {
		buf.WriteString(fmt.Sprintf("func(o %s) MarshalJSON() ([]byte, error) {\n", tname))
		buf.WriteString("var buf bytes.Buffer\n")
		buf.WriteString("_, err := o.WriteJSON(&buf)\n")
		buf.WriteString("return buf.Bytes(), err\n")
		buf.WriteString("}\n\n")
	}
	buf.WriteString(fmt.Sprintf("func(o %s) WriteJSON(w io.Writer) (int, error) {\n", tname))
	buf.WriteString("buf := jsonGetBuffer()\n")
	if err := jsonValue(typ, nil, "o", opts, buf); err != nil {
		return err
	}
	buf.WriteString("n, err := w.Write(buf.Bytes())\n")
	buf.WriteString("jsonPutBuffer(buf)\n")
	buf.WriteString("return n, err\n")
	buf.WriteString("}\n\n")
	return nil
}

func jsonStruct(st *types.Struct, p types.Type, name string, opts *Options, buf *bytes.Buffer) error {
	buf.WriteString("buf.WriteByte('{')\n")
	count := st.NumFields()
	hasFields := false
	for ii := 0; ii < count; ii++ {
		field := st.Field(ii)
		if field.IsExported() {<commit_msg>Use the functions in genutil for listing named types<commit_after>// Package json generates methods for encoding/decoding types to/from JSON.
//
// When used correctly, these methods can easily give a ~200-300% performance
// increase when serializing objects to JSON while also reducing memory usage
// by ~95-99%. For taking advantage of these gains, you must use
// gnd.la/mux/serialize or Context.WriteJson to encode to JSON, since
// json.Marshal won't use these methods correctly and might even have worse
// performance when these methods are implemented.
//
// This is a small benchmark comparing the performance of these JSON encoding
// methods. JSONDirect uses WriteJSON(), JSONSerialize uses
// gnd.la/mux/serialize (which adds some overhead because it also sets the
// Content-Length and Content-Encoding headers and thus must encode into an
// intermediate buffer first), while JSON uses json.Marshal(). All three
// benchmarks write the result to ioutil.Discard.
//
//  BenchmarkJSONDirect	    1000000 1248 ns/op	117.73 MB/s 16 B/op	2 allocs/op
//  BenchmarkJSONSerialize  1000000 1587 ns/op	92.62 MB/s  16 B/op	2 allocs/op
//  BenchmarkJSON	    500000  4583 ns/op	32.07 MB/s  620 B/op	4 allocs/op
//
// Code geerated by this package respects json related struct tags and also
// supports encoding time.Time structs directly (they're encoded as a UTC unix time).
//
// The recommended way use to generate JSON methods for a given package is
// using the gondola command rather than using this package directly.
package json

import (
	"bytes"
	"code.google.com/p/go.tools/go/types"
	"fmt"
	"gnd.la/gen/genutil"
	"gnd.la/log"
	gtypes "gnd.la/types"
	"path/filepath"
	"regexp"
)

type Method struct {
	Key       string
	Name      string
	OmitEmpty bool
}

// Options specify the options used when generating JSON related
// methods.
type Options struct {
	// Wheter to generate a MarshalJSON method. This is false by default
	// because in most cases will result in lower performance when using
	// json.Marshal, since the encoder from encoding/json will revalidate
	// the returned JSON, resulting in a performance loss. Turn this on
	// only if you're using the Methods feature (otherwise you'll get
	// different results when serializing with json.Marshal).
	MarshalJSON bool
	// Methods indicates struct methods which should be included in the JSON
	// output. The key in the map is the type name in the package (e.g.
	// MyStruct not mypackage.MyStruct).
	Methods map[string][]*Method
	// If not nil, only types matching this regexp will be included.
	Include *regexp.Regexp
	// If not nil, types matching this regexp will be excluded.
	Exclude *regexp.Regexp
}

// Gen generates a WriteJSON method and, optionally, MarshalJSON for every
// exported type in the given package. The package might be either an
// absolute path or an import path.
func Gen(pkgName string, opts *Options) error {
	pkg, err := genutil.NewPackage(pkgName)
	if err != nil {
		return err
	}
	var buf bytes.Buffer
	buf.WriteString(fmt.Sprintf("package %s\n\n", pkg.Name()))
	buf.WriteString(genutil.AutogenString())
	buf.WriteString("\nimport (\n")
	imports := []string{"bytes", "io", "runtime", "strconv", "unicode/utf8"}
	for _, v := range imports {
		buf.WriteString(fmt.Sprintf("%q\n", v))
	}
	buf.WriteString(")\n")
	buf.WriteString("var _ = strconv.FormatBool\n")
	var include *regexp.Regexp
	var exclude *regexp.Regexp
	if opts != nil {
		include = opts.Include
		exclude = opts.Exclude
	}
	var methods bytes.Buffer
	for _, v := range pkg.ExportedTypes(include, exclude) {
		methods.Reset()
		if err := jsonMarshal(v, opts, &methods); err != nil {
			log.Warningf("Skipping type %s: %s", v.Obj().Name(), err)
			continue
		}
		buf.WriteString(methods.String())
	}
	buf.WriteString(encode_go)
	buf.WriteString(buffer_go)
	out := filepath.Join(pkg.Dir(), "gen_json.go")
	log.Debugf("Writing autogenerated JSON methods to %s", out)
	return genutil.WriteAutogen(out, buf.Bytes())
}

func jsonMarshal(typ *types.Named, opts *Options, buf *bytes.Buffer) error {
	tname := typ.Obj().Name()
	if _, ok := typ.Underlying().(*types.Struct); ok {
		tname = "*" + tname
	}
	if opts != nil && opts.MarshalJSON {
		buf.WriteString(fmt.Sprintf("func(o %s) MarshalJSON() ([]byte, error) {\n", tname))
		buf.WriteString("var buf bytes.Buffer\n")
		buf.WriteString("_, err := o.WriteJSON(&buf)\n")
		buf.WriteString("return buf.Bytes(), err\n")
		buf.WriteString("}\n\n")
	}
	buf.WriteString(fmt.Sprintf("func(o %s) WriteJSON(w io.Writer) (int, error) {\n", tname))
	buf.WriteString("buf := jsonGetBuffer()\n")
	if err := jsonValue(typ, nil, "o", opts, buf); err != nil {
		return err
	}
	buf.WriteString("n, err := w.Write(buf.Bytes())\n")
	buf.WriteString("jsonPutBuffer(buf)\n")
	buf.WriteString("return n, err\n")
	buf.WriteString("}\n\n")
	return nil
}

func jsonStruct(st *types.Struct, p types.Type, name string, opts *Options, buf *bytes.Buffer) error {
	buf.WriteString("buf.WriteByte('{')\n")
	count := st.NumFields()
	hasFields := false
	for ii := 0; ii < count; ii++ {
		field := st.Field(ii)
		if field.IsExported() {
{'fraction_non_alphanumeric': 0.12874194770746494, 'fraction_numerical': 0.012694202349374763, 'mean_word_length': 3.8205479452054796, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1018752', 'n_tokens_mistral': 3520, 'n_tokens_neox': 3338, 'n_words': 1156}