<commit_before>package gog

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

type Doc struct {
	*DefKey

	Format string
	Data   string

	File string    `json:",omitempty"`
	Span [2]uint32 `json:",omitempty"`
}

func parseFiles(fset *token.FileSet, filenames []string) (map[string]*ast.File, error) {
	files := make(map[string]*ast.File)
	for _, path := range filenames {
		// read file contents using go/build context so we use our vfs if
		// present
		var f io.ReadCloser
		var err error
		if build.Default.OpenFile != nil {
			f, err = build.Default.OpenFile(path)
		} else {
			f, err = os.Open(path)
		}
		if err != nil {
			return nil, err
		}
		defer f.Close()

		file, err := parser.ParseFile(fset, path, f, parser.ParseComments)
		if err != nil {
			return nil, err
		}
		files[path] = file
	}
	return files, nil
}

func (g *Grapher) emitDocs(pkgInfo *loader.PackageInfo) error {
	objOf := make(map[token.Position]types.Object, len(pkgInfo.Defs))
	for ident, obj := range pkgInfo.Defs {
		objOf[g.program.Fset.Position(ident.Pos())] = obj
	}

	var filenames []string
	for _, f := range pkgInfo.Files {
		name := g.program.Fset.Position(f.Name.Pos()).Filename
		if filepath.Base(name) == "C" {
			// skip cgo-generated file
			continue
		}
		filenames = append(filenames, name)
	}
	sort.Strings(filenames)
	files, err := parseFiles(g.program.Fset, filenames)
	if err != nil {
		return err
	}

	// First we collect all of the Doc comments from the files,
	// which will make up the Doc for the package. If more than
	// one file has a doc associated with it, append them
	// together.
	pkgDoc := ""
	for _, f := range files {
		if f.Doc == nil {
			continue
		}
		if pkgDoc == "" {
			pkgDoc = f.Doc.Text()
			continue
		}
		pkgDoc += "\n" + f.Doc.Text()
	}
	g.emitDoc(types.NewPkgName(0, pkgInfo.Pkg, pkgInfo.Pkg.Path(), pkgInfo.Pkg), nil, pkgDoc, "")

	// We walk the AST for comments attached to nodes.
	for filename, f := range files {
		if f.Doc != nil {
			g.emitDoc(nil, f.Doc, f.Doc.Text(), filename)
		}
		// docSeen is a map from the starting byte of a doc to
		// an empty struct.
		docSeen := make(map[token.Pos]struct{})
		ast.Inspect(f, func(node ast.Node) bool {
			switch n := node.(type) {
			case *ast.Field:
				if n.Doc == nil {
					return true
				}
				for _, i := range n.Names {
					if g.emitDoc(objOf[g.program.Fset.Position(i.Pos())], n.Doc, n.Doc.Text(), filename) {
						docSeen[n.Doc.Pos()] = struct{}{}
					}<commit_msg>only emit file doc once<commit_after>package gog

import (
	"bytes"
	"go/ast"
	"go/build"
	"go/doc"
	"go/parser"
	"go/token"
	"io"
	"os"
	"path/filepath"
	"sort"

	"golang.org/x/tools/go/loader"
	"golang.org/x/tools/go/types"
)

type Doc struct {
	*DefKey

	Format string
	Data   string

	File string    `json:",omitempty"`
	Span [2]uint32 `json:",omitempty"`
}

func parseFiles(fset *token.FileSet, filenames []string) (map[string]*ast.File, error) {
	files := make(map[string]*ast.File)
	for _, path := range filenames {
		// read file contents using go/build context so we use our vfs if
		// present
		var f io.ReadCloser
		var err error
		if build.Default.OpenFile != nil {
			f, err = build.Default.OpenFile(path)
		} else {
			f, err = os.Open(path)
		}
		if err != nil {
			return nil, err
		}
		defer f.Close()

		file, err := parser.ParseFile(fset, path, f, parser.ParseComments)
		if err != nil {
			return nil, err
		}
		files[path] = file
	}
	return files, nil
}

func (g *Grapher) emitDocs(pkgInfo *loader.PackageInfo) error {
	objOf := make(map[token.Position]types.Object, len(pkgInfo.Defs))
	for ident, obj := range pkgInfo.Defs {
		objOf[g.program.Fset.Position(ident.Pos())] = obj
	}

	var filenames []string
	for _, f := range pkgInfo.Files {
		name := g.program.Fset.Position(f.Name.Pos()).Filename
		if filepath.Base(name) == "C" {
			// skip cgo-generated file
			continue
		}
		filenames = append(filenames, name)
	}
	sort.Strings(filenames)
	files, err := parseFiles(g.program.Fset, filenames)
	if err != nil {
		return err
	}

	// First we collect all of the Doc comments from the files,
	// which will make up the Doc for the package. If more than
	// one file has a doc associated with it, append them
	// together.
	pkgDoc := ""
	for _, f := range files {
		if f.Doc == nil {
			continue
		}
		if pkgDoc == "" {
			pkgDoc = f.Doc.Text()
			continue
		}
		pkgDoc += "\n" + f.Doc.Text()
	}
	g.emitDoc(types.NewPkgName(0, pkgInfo.Pkg, pkgInfo.Pkg.Path(), pkgInfo.Pkg), nil, pkgDoc, "")

	// We walk the AST for comments attached to nodes.
	for filename, f := range files {
		// docSeen is a map from the starting byte of a doc to
		// an empty struct.
		docSeen := make(map[token.Pos]struct{})
		ast.Inspect(f, func(node ast.Node) bool {
			switch n := node.(type) {
			case *ast.Field:
				if n.Doc == nil {
					return true
				}
				for _, i := range n.Names {
					if g.emitDoc(objOf[g.program.Fset.Position(i.Pos())], n.Doc, n.Doc.Text(), filename) {
						docSeen[n.Doc.Pos()] = struct{}{}
					}
{'fraction_non_alphanumeric': 0.15120140652471187, 'fraction_numerical': 0.0015628052353975385, 'mean_word_length': 3.015686274509804, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3037120', 'n_tokens_mistral': 2122, 'n_tokens_neox': 1922, 'n_words': 519}