<commit_before>package main

import (
	"strings"

	. "github.com/davelondon/jennifer/jen"
)

func main() {

	file := NewFile("jen")

	file.Comment("\tThis file is generated by genjen - do not edit!\n")
	file.Line()

	// This generates a function for each method. The
	// funciton creates a new statement and calls the
	// method.
	redirect := func(
		name string,
		comment Code,
		params []Code,
		call []Code,
	) {
		/*
			// {comment}
			func {name}({params}) *Group {
				return newStatement().{name}({call})
			}
		*/
		file.Add(comment)
		file.Func().Id(name).Params(
			params...,
		).Op("*").Id("Group").Block(
			Return().Id("newStatement").Call().Op(".").Id(name).Call(
				call...,
			),
		)
	}

	for _, b := range Blocks {
		b := b // b used in closures
		if b.Name == "" {
			continue
		}
		comment := Commentf("%s inserts %s", b.Name, b.Desc)

		redirect(
			b.Name,
			comment,
			[]Code{Id("c").Op("...").Id("Code")},
			[]Code{Id("c").Op("...")},
		)

		/*
			func (g *Group) {Name}(c ...Code) *Group {
				if startNewStatement(g.syntax) {
					s := {Name}(c...)
					g.items = append(g.items, s)
					return s
				}
				s := Group{
					syntax: {Syntax},
					items:  c,
				}
				g.items = append(g.items, s)
				return g
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(b.Name).Params(
			Id("c").Op("...").Id("Code"),
		).Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(
				Id("g", "syntax"),
			).Block(
				Id("s").Op(":=").Id(b.Name).Call(
					Id("c").Op("..."),
				),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Id("s").Op(":=").Op("&").Id("Group").Dict(map[Code]Code{
				Id("syntax"): Id(b.Syntax),
				Id("items"):  Id("c"),
			}),
			Id("g", "items").Op("=").Append(
				Id("g", "items"),
				Id("s"),
			),
			Return(Id("g")),
		)

		nameFunc := b.Name + "Func"
		redirect(
			nameFunc,
			comment,
			[]Code{Id("f").Func().Params(Op("*").Id("Group"))},
			[]Code{Id("f")},
		)

		/*
			func (g *Group) {NameFunc}(f func(*Group)) *Group {
				if startNewStatement(g.syntax) {
					s := {NameFunc}(f)
					g.items = append(g.items, s)
					return s
				}
				s := &Group{
					syntax:    {Syntax},
				}
				f(s)
				g.items = append(g.items, s)
				return g
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(nameFunc).Params(
			Id("f").Func().Params(Op("*").Id("Group")),
		).Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(
				Id("g", "syntax"),
			).Block(
				Id("s").Op(":=").Id(nameFunc).Call(
					Id("f"),
				),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Id("s").Op(":=").Op("&").Id("Group").Dict(map[Code]Code{
				Id("syntax"): Id(b.Syntax),
			}),
			Id("f").Call(Id("s")),
			Id("g", "items").Op("=").Append(
				Id("g", "items"),
				Id("s"),
			),
			Return(Id("g")),
		)
	}

	type tkn struct {
		name      string
		cap       string
		tokenType string
		tokenDesc string
	}
	tokens := []tkn{}
	for _, v := range Identifiers {
		tokens = append(tokens, tkn{
			name:      v,
			cap:       strings.ToUpper(v[:1]) + v[1:],
			tokenType: "identifierToken",
			tokenDesc: "identifier",
		})
	}
	for _, v := range Keywords {
		tokens = append(tokens, tkn{
			name:      v,
			cap:       strings.ToUpper(v[:1]) + v[1:],
			tokenType: "keywordToken",
			tokenDesc: "keyword",
		})
	}

	for _, t := range tokens {
		t := t // used in closures
		comment := Commentf(
			"%s inserts the %s %s",
			t.cap,
			t.name,
			t.tokenDesc,
		)
		redirect(
			t.cap,
			comment,
			[]Code{},
			[]Code{},
		)

		/*
			func (g *Group) {Name}() *Group {
				if startNewStatement(g.syntax) {
					s := {Name}()
					g.items = append(g.items, s)
					return s
				}
				t := token{
					Group:    g,
					typ:     {identifierToken|keywordToken},
					content: "{Name}",
				}
				g.items = append(g.items, t)
				return g
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(t.cap).Params().Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(
				Id("g", "syntax"),
			).Block(
				Id("s").Op(":=").Id(t.cap).Call(),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Id("t").Op(":=").Id("token").Dict(map[Code]Code{
				Id("Group"):   Id("g"),
				Id("typ"):     Id(t.tokenType),
				Id("content"): Lit(t.name),
			}),
			Id("g", "items").Op("=").Append(
				Id("g", "items"),
				Id("t"),
			),
			Return(Id("g")),
		)
	}

	for _, f := range Functions {
		f := f // used in closure
		capName := strings.ToUpper(f[:1]) + f[1:]
		comment := Commentf(
			"%s inserts the built in function %s",
			capName,
			f,
		)

		redirect(
			capName,
			comment,
			[]Code{Id("c").Op("...").Id("Code")},
			[]Code{Id("c").Op("...")},
		)

		/*
			func (g *Group) {Name}(c ...Code) *Group {
				if startNewStatement(g.syntax) {
					s := {Name}(c...)
					g.items = append(g.items, s)
					return s
				}
				return g.Id("{Name}").{Call|List}(c...)
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(capName).Params(
			Id("c").Op("...").Id("Code"),
		).Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(Id("g", "syntax")).Block(
				Id("s").Op(":=").Id(capName).Call(
					Id("c").Op("..."),
				),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Return().Id("g", "Id").Call(
				Lit(f),
			).Op(".").Id("Call").Call(
				Id("c").Op("..."),
			),
		)
	}

	err := file.Save("./generated.go")
	//err := file.Render(os.Stdout)
	if err != nil {
		panic(err)
	}

}
<commit_msg>nil instead of []Code{}<commit_after>package main

import (
	"strings"

	. "github.com/davelondon/jennifer/jen"
)

func main() {

	file := NewFile("jen")

	file.Comment("\tThis file is generated by genjen - do not edit!\n")
	file.Line()

	// This generates a function for each method. The
	// funciton creates a new statement and calls the
	// method.
	redirect := func(
		name string,
		comment Code,
		params []Code,
		call []Code,
	) {
		/*
			// {comment}
			func {name}({params}) *Group {
				return newStatement().{name}({call})
			}
		*/
		file.Add(comment)
		file.Func().Id(name).Params(
			params...,
		).Op("*").Id("Group").Block(
			Return().Id("newStatement").Call().Op(".").Id(name).Call(
				call...,
			),
		)
	}

	for _, b := range Blocks {
		b := b // b used in closures
		if b.Name == "" {
			continue
		}
		comment := Commentf("%s inserts %s", b.Name, b.Desc)

		redirect(
			b.Name,
			comment,
			[]Code{Id("c").Op("...").Id("Code")},
			[]Code{Id("c").Op("...")},
		)

		/*
			func (g *Group) {Name}(c ...Code) *Group {
				if startNewStatement(g.syntax) {
					s := {Name}(c...)
					g.items = append(g.items, s)
					return s
				}
				s := Group{
					syntax: {Syntax},
					items:  c,
				}
				g.items = append(g.items, s)
				return g
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(b.Name).Params(
			Id("c").Op("...").Id("Code"),
		).Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(
				Id("g", "syntax"),
			).Block(
				Id("s").Op(":=").Id(b.Name).Call(
					Id("c").Op("..."),
				),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Id("s").Op(":=").Op("&").Id("Group").Dict(map[Code]Code{
				Id("syntax"): Id(b.Syntax),
				Id("items"):  Id("c"),
			}),
			Id("g", "items").Op("=").Append(
				Id("g", "items"),
				Id("s"),
			),
			Return(Id("g")),
		)

		nameFunc := b.Name + "Func"
		redirect(
			nameFunc,
			comment,
			[]Code{Id("f").Func().Params(Op("*").Id("Group"))},
			[]Code{Id("f")},
		)

		/*
			func (g *Group) {NameFunc}(f func(*Group)) *Group {
				if startNewStatement(g.syntax) {
					s := {NameFunc}(f)
					g.items = append(g.items, s)
					return s
				}
				s := &Group{
					syntax:    {Syntax},
				}
				f(s)
				g.items = append(g.items, s)
				return g
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(nameFunc).Params(
			Id("f").Func().Params(Op("*").Id("Group")),
		).Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(
				Id("g", "syntax"),
			).Block(
				Id("s").Op(":=").Id(nameFunc).Call(
					Id("f"),
				),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Id("s").Op(":=").Op("&").Id("Group").Dict(map[Code]Code{
				Id("syntax"): Id(b.Syntax),
			}),
			Id("f").Call(Id("s")),
			Id("g", "items").Op("=").Append(
				Id("g", "items"),
				Id("s"),
			),
			Return(Id("g")),
		)
	}

	type tkn struct {
		name      string
		cap       string
		tokenType string
		tokenDesc string
	}
	tokens := []tkn{}
	for _, v := range Identifiers {
		tokens = append(tokens, tkn{
			name:      v,
			cap:       strings.ToUpper(v[:1]) + v[1:],
			tokenType: "identifierToken",
			tokenDesc: "identifier",
		})
	}
	for _, v := range Keywords {
		tokens = append(tokens, tkn{
			name:      v,
			cap:       strings.ToUpper(v[:1]) + v[1:],
			tokenType: "keywordToken",
			tokenDesc: "keyword",
		})
	}

	for _, t := range tokens {
		t := t // used in closures
		comment := Commentf(
			"%s inserts the %s %s",
			t.cap,
			t.name,
			t.tokenDesc,
		)
		redirect(
			t.cap,
			comment,
			nil,
			nil,
		)

		/*
			func (g *Group) {Name}() *Group {
				if startNewStatement(g.syntax) {
					s := {Name}()
					g.items = append(g.items, s)
					return s
				}
				t := token{
					Group:    g,
					typ:     {identifierToken|keywordToken},
					content: "{Name}",
				}
				g.items = append(g.items, t)
				return g
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(t.cap).Params().Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(
				Id("g", "syntax"),
			).Block(
				Id("s").Op(":=").Id(t.cap).Call(),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Id("t").Op(":=").Id("token").Dict(map[Code]Code{
				Id("Group"):   Id("g"),
				Id("typ"):     Id(t.tokenType),
				Id("content"): Lit(t.name),
			}),
			Id("g", "items").Op("=").Append(
				Id("g", "items"),
				Id("t"),
			),
			Return(Id("g")),
		)
	}

	for _, f := range Functions {
		f := f // used in closure
		capName := strings.ToUpper(f[:1]) + f[1:]
		comment := Commentf(
			"%s inserts the built in function %s",
			capName,
			f,
		)

		redirect(
			capName,
			comment,
			[]Code{Id("c").Op("...").Id("Code")},
			[]Code{Id("c").Op("...")},
		)

		/*
			func (g *Group) {Name}(c ...Code) *Group {
				if startNewStatement(g.syntax) {
					s := {Name}(c...)
					g.items = append(g.items, s)
					return s
				}
				return g.Id("{Name}").{Call|List}(c...)
			}
		*/
		file.Add(comment)
		file.Func().Params(
			Id("g").Op("*").Id("Group"),
		).Id(capName).Params(
			Id("c").Op("...").Id("Code"),
		).Op("*").Id("Group").Block(
			If().Id("startNewStatement").Call(Id("g", "syntax")).Block(
				Id("s").Op(":=").Id(capName).Call(
					Id("c").Op("..."),
				),
				Id("g", "items").Op("=").Append(
					Id("g", "items"),
					Id("s"),
				),
				Return(Id("s")),
			),
			Return().Id("g", "Id").Call(
				Lit(f),
			).Op(".").Id("Call").Call(
				Id("c").Op("..."),
			),
		)
	}

	err := file.Save("./generated.go")
	//err := file.Render(os.Stdout)
	if err != nil {
		panic(err)
	}

}

{'fraction_non_alphanumeric': 0.27151283830673145, 'fraction_numerical': 0.001040943789035392, 'mean_word_length': 2.4642427884615383, 'pattern_counts': {'":': 14, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5472596', 'n_tokens_mistral': 5696, 'n_tokens_neox': 4575, 'n_words': 795}