<commit_before>package main

import (
	"go/ast"
	"go/token"
	"fmt"
	"reflect"
	"io"
	"bytes"
	"strings"
)

// Decl.Class
const (
	DECL_CONST = iota
	DECL_VAR
	DECL_TYPE
	DECL_FUNC
	DECL_MODULE

	// this one serves as a temporary type for those methods that were
	// declared before their actual owner
	DECL_METHODS_STUB
)

// Decl.Flags
const (
	DECL_FOREIGN = 1 << iota // imported from another module
)

var declClassToString = [...]string{
	DECL_CONST:        "const",
	DECL_VAR:          "var",
	DECL_TYPE:         "type",
	DECL_FUNC:         "func",
	DECL_MODULE:       "module",
	DECL_METHODS_STUB: "IF YOU SEE THIS, REPORT A BUG", // :D
}

var declClassToStringDebug = [...]string{
	DECL_CONST:        " const",
	DECL_VAR:          "   var",
	DECL_TYPE:         "  type",
	DECL_FUNC:         "  func",
	DECL_MODULE:       "module",
	DECL_METHODS_STUB: "  stub",
}

type Decl struct {
	Name  string
	Type  ast.Expr
	Class int16
	Flags int16

	// functions for interface type, fields+methods for struct type
	// variables with anonymous struct/interface type may have children too
	Children map[string]*Decl

	// embedded types
	Embedded []ast.Expr

	// if the type is unknown at AST building time, I'm using these
	Value ast.Expr

	// if it's a multiassignment and the Value is a CallExpr, it is being set
	// to an index into the return value tuple, otherwise it's a -1
	ValueIndex int

	// scope where this Decl was declared in (not its visibilty scope!)
	// Decl uses it for type inference
	Scope *Scope
}

func astDeclType(d ast.Decl) ast.Expr {
	switch t := d.(type) {
	case *ast.GenDecl:
		switch t.Tok {
		case token.CONST, token.VAR:
			c := t.Specs[0].(*ast.ValueSpec)
			return c.Type
		case token.TYPE:
			t := t.Specs[0].(*ast.TypeSpec)
			return t.Type
		}
	case *ast.FuncDecl:
		return t.Type
	}
	panic("unreachable")
	return nil
}

func astDeclClass(d ast.Decl) int {
	switch t := d.(type) {
	case *ast.GenDecl:
		switch t.Tok {
		case token.VAR:
			return DECL_VAR
		case token.CONST:
			return DECL_CONST
		case token.TYPE:
			return DECL_TYPE
		}
	case *ast.FuncDecl:
		return DECL_FUNC
	}
	panic("unreachable")
	return 0
}

func astDeclConvertable(d ast.Decl) bool {
	switch t := d.(type) {
	case *ast.GenDecl:
		switch t.Tok {
		case token.VAR, token.CONST, token.TYPE:
			return true
		}
	case *ast.FuncDecl:
		return true
	}
	return false
}

func astFieldListToDecls(f *ast.FieldList, class int, flags int, scope *Scope) map[string]*Decl {
	count := 0
	for _, field := range f.List {
		count += len(field.Names)
	}

	if count == 0 {
		return nil
	}

	decls := make(map[string]*Decl, count)
	i := 0
	for _, field := range f.List {
		for _, name := range field.Names {
			if flags&DECL_FOREIGN != 0 && !ast.IsExported(name.Name()) {
				continue
			}
			d := new(Decl)
			d.Name = name.Name()
			d.Type = field.Type
			d.Class = int16(class)
			d.Children = astTypeToChildren(field.Type, flags, scope)
			d.Embedded = astTypeToEmbedded(field.Type)
			d.Scope = scope
			d.ValueIndex = -1
			d.init()
			decls[d.Name] = d
			i++
		}
	}
	return decls
}

func astFieldListToEmbedded(f *ast.FieldList) []ast.Expr {
	count := 0<commit_msg>Forget to apply parent's flags to children.<commit_after>package main

import (
	"go/ast"
	"go/token"
	"fmt"
	"reflect"
	"io"
	"bytes"
	"strings"
)

// Decl.Class
const (
	DECL_CONST = iota
	DECL_VAR
	DECL_TYPE
	DECL_FUNC
	DECL_MODULE

	// this one serves as a temporary type for those methods that were
	// declared before their actual owner
	DECL_METHODS_STUB
)

// Decl.Flags
const (
	DECL_FOREIGN = 1 << iota // imported from another module
)

var declClassToString = [...]string{
	DECL_CONST:        "const",
	DECL_VAR:          "var",
	DECL_TYPE:         "type",
	DECL_FUNC:         "func",
	DECL_MODULE:       "module",
	DECL_METHODS_STUB: "IF YOU SEE THIS, REPORT A BUG", // :D
}

var declClassToStringDebug = [...]string{
	DECL_CONST:        " const",
	DECL_VAR:          "   var",
	DECL_TYPE:         "  type",
	DECL_FUNC:         "  func",
	DECL_MODULE:       "module",
	DECL_METHODS_STUB: "  stub",
}

type Decl struct {
	Name  string
	Type  ast.Expr
	Class int16
	Flags int16

	// functions for interface type, fields+methods for struct type
	// variables with anonymous struct/interface type may have children too
	Children map[string]*Decl

	// embedded types
	Embedded []ast.Expr

	// if the type is unknown at AST building time, I'm using these
	Value ast.Expr

	// if it's a multiassignment and the Value is a CallExpr, it is being set
	// to an index into the return value tuple, otherwise it's a -1
	ValueIndex int

	// scope where this Decl was declared in (not its visibilty scope!)
	// Decl uses it for type inference
	Scope *Scope
}

func astDeclType(d ast.Decl) ast.Expr {
	switch t := d.(type) {
	case *ast.GenDecl:
		switch t.Tok {
		case token.CONST, token.VAR:
			c := t.Specs[0].(*ast.ValueSpec)
			return c.Type
		case token.TYPE:
			t := t.Specs[0].(*ast.TypeSpec)
			return t.Type
		}
	case *ast.FuncDecl:
		return t.Type
	}
	panic("unreachable")
	return nil
}

func astDeclClass(d ast.Decl) int {
	switch t := d.(type) {
	case *ast.GenDecl:
		switch t.Tok {
		case token.VAR:
			return DECL_VAR
		case token.CONST:
			return DECL_CONST
		case token.TYPE:
			return DECL_TYPE
		}
	case *ast.FuncDecl:
		return DECL_FUNC
	}
	panic("unreachable")
	return 0
}

func astDeclConvertable(d ast.Decl) bool {
	switch t := d.(type) {
	case *ast.GenDecl:
		switch t.Tok {
		case token.VAR, token.CONST, token.TYPE:
			return true
		}
	case *ast.FuncDecl:
		return true
	}
	return false
}

func astFieldListToDecls(f *ast.FieldList, class int, flags int, scope *Scope) map[string]*Decl {
	count := 0
	for _, field := range f.List {
		count += len(field.Names)
	}

	if count == 0 {
		return nil
	}

	decls := make(map[string]*Decl, count)
	i := 0
	for _, field := range f.List {
		for _, name := range field.Names {
			if flags&DECL_FOREIGN != 0 && !ast.IsExported(name.Name()) {
				continue
			}
			d := new(Decl)
			d.Name = name.Name()
			d.Type = field.Type
			d.Class = int16(class)
			d.Flags = int16(flags)
			d.Children = astTypeToChildren(field.Type, flags, scope)
			d.Embedded = astTypeToEmbedded(field.Type)
			d.Scope = scope
			d.ValueIndex = -1
			d.init()
			decls[d.Name] = d
			i++
		}
	}
	return decls
}

func astFieldListToEmbedded(f *ast.FieldList) []ast.Expr {
	count := 0
{'fraction_non_alphanumeric': 0.12013801756587202, 'fraction_numerical': 0.0056461731493099125, 'mean_word_length': 2.929143561306223, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2900164', 'n_tokens_mistral': 2492, 'n_tokens_neox': 2304, 'n_words': 671}