<commit_before>// Copyright 2013 The rspace Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc is a simple document printer that produces the doc comments
// for its argument symbols, using a more Go-like UI than godoc.
// It can also search for symbols by looking in all packages, and
// case is ignored. For instance:
//	doc isupper
// will find unicode.IsUpper.
//
// Usage:
//	doc pkg.name   # "doc io.Writer"
//	doc pkg name   # "doc fmt Printf"
//	doc name       # "doc isupper" (finds unicode.IsUpper)
//
// The pkg is the last element of the package path;
// no slashes (ast.Node not go/ast.Node).
//
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"unicode"
	"unicode/utf8"
)

const usageDoc = `Find documentation for names.
usage:
	doc pkg.name   # "doc io.Writer"
	doc pkg name   # "doc fmt Printf"
	doc name       # "doc isupper" finds unicode.IsUpper
pkg is the last component of any package, e.g. fmt, parser
name is the name of an exported symbol; case is ignored in matches.
`

func usage() {
	fmt.Fprintf(os.Stderr, usageDoc)
	os.Exit(2)
}

func main() {
	flag.Parse()
	var pkg, name string
	switch flag.NArg() {
	case 1:
		if strings.Contains(flag.Arg(0), ".") {
			pkg, name = split(flag.Arg(0))
		} else {
			name = flag.Arg(0)
		}
	case 2:
		pkg, name = flag.Arg(0), flag.Arg(1)
	default:
		usage()
	}
	for _, path := range paths(pkg) {
		lookInDirectory(path, name)
	}
}

func split(arg string) (pkg, name string) {
	str := strings.Split(arg, ".")
	if len(str) != 2 {
		usage()
	}
	return str[0], str[1]
}

func paths(pkg string) []string {
	pkgs := pathsFor(runtime.GOROOT(), pkg)
	gopath := os.Getenv("GOPATH")
	if gopath != "" {
		for _, root := range splitGopath(gopath) {
			pkgs = append(pkgs, pathsFor(root, pkg)...)
		}
	}
	return pkgs
}

func splitGopath(gopath string) []string {
	return strings.Split(gopath, string(os.PathListSeparator))
}

// pathsFor recursively walks the tree looking for possible directories for the package:
// those whose basename is pkg.
func pathsFor(root, pkg string) []string {
	root = path.Join(root, "src")
	pkgPaths := make([]string, 0, 10)
	visit := func(pathName string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		// One package per directory. Ignore the files themselves.
		if !f.IsDir() {
			return nil
		}
		// No .hg or other dot nonsense please.
		if strings.Contains(pathName, "/.") { // TODO: Unix-specific?
			return filepath.SkipDir
		}
		// Is the last element of the path correct
		if pkg == "" || filepath.Base(pathName) == pkg {
			pkgPaths = append(pkgPaths, pathName)
		}
		return nil
	}

	filepath.Walk(root, visit)
	return pkgPaths
}

// lookInDirectory looks in the package (if any) in the directory for the named exported identifier.
func lookInDirectory(directory, name string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		// If it's just that there are no go source files, that's fine.
		if _, nogo := err.(*build.NoGoError); nogo {
			return<commit_msg>doc: remove one more unixism, fix a TODO<commit_after>// Copyright 2013 The rspace Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// Doc is a simple document printer that produces the doc comments
// for its argument symbols, using a more Go-like UI than godoc.
// It can also search for symbols by looking in all packages, and
// case is ignored. For instance:
//	doc isupper
// will find unicode.IsUpper.
//
// Usage:
//	doc pkg.name   # "doc io.Writer"
//	doc pkg name   # "doc fmt Printf"
//	doc name       # "doc isupper" (finds unicode.IsUpper)
//
// The pkg is the last element of the package path;
// no slashes (ast.Node not go/ast.Node).
//
package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/build"
	"go/parser"
	"go/printer"
	"go/token"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"runtime"
	"strings"
	"unicode"
	"unicode/utf8"
)

const usageDoc = `Find documentation for names.
usage:
	doc pkg.name   # "doc io.Writer"
	doc pkg name   # "doc fmt Printf"
	doc name       # "doc isupper" finds unicode.IsUpper
pkg is the last component of any package, e.g. fmt, parser
name is the name of an exported symbol; case is ignored in matches.
`

func usage() {
	fmt.Fprintf(os.Stderr, usageDoc)
	os.Exit(2)
}

func main() {
	flag.Parse()
	var pkg, name string
	switch flag.NArg() {
	case 1:
		if strings.Contains(flag.Arg(0), ".") {
			pkg, name = split(flag.Arg(0))
		} else {
			name = flag.Arg(0)
		}
	case 2:
		pkg, name = flag.Arg(0), flag.Arg(1)
	default:
		usage()
	}
	for _, path := range paths(pkg) {
		lookInDirectory(path, name)
	}
}

func split(arg string) (pkg, name string) {
	str := strings.Split(arg, ".")
	if len(str) != 2 {
		usage()
	}
	return str[0], str[1]
}

func paths(pkg string) []string {
	pkgs := pathsFor(runtime.GOROOT(), pkg)
	gopath := os.Getenv("GOPATH")
	if gopath != "" {
		for _, root := range splitGopath(gopath) {
			pkgs = append(pkgs, pathsFor(root, pkg)...)
		}
	}
	return pkgs
}

func splitGopath(gopath string) []string {
	return strings.Split(gopath, string(os.PathListSeparator))
}

var slashDot = string(filepath.Separator) + "."

// pathsFor recursively walks the tree looking for possible directories for the package:
// those whose basename is pkg.
func pathsFor(root, pkg string) []string {
	root = path.Join(root, "src")
	pkgPaths := make([]string, 0, 10)
	visit := func(pathName string, f os.FileInfo, err error) error {
		if err != nil {
			return err
		}
		// One package per directory. Ignore the files themselves.
		if !f.IsDir() {
			return nil
		}
		// No .hg or other dot nonsense please.
		if strings.Contains(pathName, slashDot) {
			return filepath.SkipDir
		}
		// Is the last element of the path correct
		if pkg == "" || filepath.Base(pathName) == pkg {
			pkgPaths = append(pkgPaths, pathName)
		}
		return nil
	}

	filepath.Walk(root, visit)
	return pkgPaths
}

// lookInDirectory looks in the package (if any) in the directory for the named exported identifier.
func lookInDirectory(directory, name string) {
	pkg, err := build.Default.ImportDir(directory, 0)
	if err != nil {
		// If it's just that there are no go source files, that's fine.
		if _, nogo := err.(*build.NoGoError); nogo {
			return
{'fraction_non_alphanumeric': 0.13213074313906875, 'fraction_numerical': 0.006167129201356768, 'mean_word_length': 3.2481990831696135, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '541167', 'n_tokens_mistral': 2405, 'n_tokens_neox': 2249, 'n_words': 772}