<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"
)

var src, dst string

func init() {
	flag.StringVar(&src, "src", "", "source file: -src filepath")
	flag.StringVar(&dst, "dst", "", "dst file: -dst filepath")
	flag.Parse()
}
func main() {

	var funcPattern string = `^func\s(\w+)\(.*?\)`
	var methodPattern string = `^func\s\(\w.*?\s(.*?)\)\s(\w+)`
	var typePattern string = `^type\s(.*?)\s\w+\s{`

	fmt.Println(src, dst)
	funcCompile, _ := regexp.Compile(funcPattern)
	methodCompile, _ := regexp.Compile(methodPattern)
	typeCompile, _ := regexp.Compile(typePattern)
	srcfile, err := os.Open(src)
	check(err)
	defer srcfile.Close()

	dstfile, err := os.Create(dst)
	check(err)
	defer srcfile.Close()
	scanner := bufio.NewScanner(srcfile)

	mflag := false

	for scanner.Scan() {

		line := scanner.Text()
		if mflag {
			fmt.Fprintln(dstfile, line)
			if line == "}" {
				fmt.Fprint(dstfile, "```\n\n")
				mflag = false
				continue
			}
		}

		if ok := funcCompile.MatchString(line); ok {
			funcName := funcCompile.FindStringSubmatch(line)
			fmt.Fprintf(dstfile, "###%s\n", funcName[0])
			withgo(dstfile, line)
			fmt.Fprint(dstfile, "```\n\n")
		}

		if ok := methodCompile.MatchString(line); ok {
			methodName := methodCompile.FindStringSubmatch(line)
			fmt.Fprintf(dstfile, "###%s\n", methodName[0])
			withgo(dstfile, line)
			fmt.Fprint(dstfile, "```\n\n")

		}

		if ok := typeCompile.MatchString(line); ok {
			typeName := typeCompile.FindStringSubmatch(line)
			fmt.Fprintf(dstfile, "###%s\n", strings.Trim(typeName[0], "{"))
			withgo(dstfile, line)
			mflag = true
		}

	}
	fmt.Println("completed!")

}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func withgo(f *os.File, line string) {<commit_msg>修改godoc2md.go，改为管道输入<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"os"
	"regexp"
	"strings"
)

var src string
var dst string

func init() {
	flag.StringVar(&dst, "dst", "", "dst file: -dst filepath")
	flag.Parse()
}
func main() {

	var funcPattern string = `^func\s(\w+)\(.*?\)`
	var methodPattern string = `^func\s\(\w.*?\s(.*?)\)\s(\w+)`
	var typePattern string = `^type\s(.*?)\s\w+\s{`

	fmt.Println(src, dst)
	funcCompile, _ := regexp.Compile(funcPattern)
	methodCompile, _ := regexp.Compile(methodPattern)
	typeCompile, _ := regexp.Compile(typePattern)

	srcfile := os.Stdin

	dstfile, err := os.Create(dst)
	check(err)
	defer dstfile.Close()

	scanner := bufio.NewScanner(srcfile)
	defer srcfile.Close()
	mflag := false

	for scanner.Scan() {

		line := scanner.Text()
		if mflag {
			fmt.Fprintln(dstfile, line)
			if line == "}" {
				fmt.Fprint(dstfile, "```\n\n")
				mflag = false
				continue
			}
		}

		if ok := funcCompile.MatchString(line); ok {
			funcName := funcCompile.FindStringSubmatch(line)
			fmt.Fprintf(dstfile, "###%s\n", funcName[0])
			withgo(dstfile, line)
			fmt.Fprint(dstfile, "```\n\n")
		}

		if ok := methodCompile.MatchString(line); ok {
			methodName := methodCompile.FindStringSubmatch(line)
			fmt.Fprintf(dstfile, "###%s\n", methodName[0])
			withgo(dstfile, line)
			fmt.Fprint(dstfile, "```\n\n")
		}

		if ok := typeCompile.MatchString(line); ok {
			typeName := typeCompile.FindStringSubmatch(line)
			fmt.Fprintf(dstfile, "###%s\n", strings.Trim(typeName[0], "{"))
			withgo(dstfile, line)
			mflag = true
		}

	}
	fmt.Println("completed!")

}

func check(err error) {
	if err != nil {
		panic(err)
	}
}

func withgo(f *os.File, line string) {
{'fraction_non_alphanumeric': 0.1912568306010929, 'fraction_numerical': 0.0020132297958009777, 'mean_word_length': 3.8105117565698476, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6758738', 'n_tokens_mistral': 1539, 'n_tokens_neox': 1380, 'n_words': 260}