<commit_before>package main

import (
	"bytes"
	"flag"
	"fmt"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"

	plugin "code.google.com/p/goprotobuf/compiler/plugin"
	"code.google.com/p/goprotobuf/proto"

	"github.com/dsymonds/gotoc/parser"
	"github.com/dsymonds/gotoc/resolver"
)

var (
	// Flags
	helpShort = flag.Bool("h", false, "Show usage text (same as --help).")
	helpLong  = flag.Bool("help", false, "Show usage text (same as -h).")

	importPath     = flag.String("import_path", ".", "Comma-separated list of paths to search for imports.")
	pluginBinary   = flag.String("plugin", "protoc-gen-go", "The code generator plugin to use.")
	descriptorOnly = flag.Bool("descriptor_only", false, "Whether to print out only the FileDescriptorSet.")
)

func fullPath(binary string, paths []string) string {
	if strings.Index(binary, "/") >= 0 {
		// path with path component
		return binary
	}
	for _, p := range paths {
		full := path.Join(p, binary)
		fi, err := os.Stat(full)
		if err == nil && !fi.IsDir() {
			return full
		}
	}
	return ""
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *helpShort || *helpLong || flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	fds, err := parser.ParseFiles(flag.Args(), strings.Split(*importPath, ","))
	if err != nil {
		log.Fatalf("Failed parsing: %v", err)
	}
	if err := resolver.ResolveSymbols(fds); err != nil {
		log.Fatalf("Failed resolving symbols: %v", err)
	}

	if *descriptorOnly {
		proto.MarshalText(os.Stdout, fds)
		os.Exit(0)
	}

	fmt.Println("-----")
	proto.MarshalText(os.Stdout, fds)
	fmt.Println("-----")

	// Prepare request.
	cgRequest := &plugin.CodeGeneratorRequest{
		FileToGenerate: flag.Args(),
		// TODO: proto_file should be topologically sorted (bottom-up)
		ProtoFile: fds.File,
	}
	buf, err := proto.Marshal(cgRequest)
	if err != nil {
		log.Fatalf("Failed marshaling CG request: %v", err)
	}

	// Find plugin.
	pluginPath := fullPath(*pluginBinary, strings.Split(os.Getenv("PATH"), ":"))
	if pluginPath == "" {
		log.Fatalf("Failed finding plugin binary %q", *pluginBinary)
	}

	// Run the plugin subprocess.
	cmd := &exec.Cmd{
		Path:   pluginPath,
		Stdin:  bytes.NewBuffer(buf),
		Stderr: os.Stderr,
	}
	buf, err = cmd.Output()
	if err != nil {
		log.Fatalf("Failed running plugin: %v", err)
	}

	// Parse the response.
	cgResponse := new(plugin.CodeGeneratorResponse)
	if err = proto.Unmarshal(buf, cgResponse); err != nil {
		log.Fatalf("Failed unmarshaling CG response: %v", err)
	}

	// TODO: check cgResponse.Error

	// TODO: write files
	for _, f := range cgResponse.File {
		fmt.Printf("--[ %v ]--\n", proto.GetString(f.Name))
		fmt.Println(proto.GetString(f.Content))
	}
	fmt.Println("-----")
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage:  %s [options] <foo.proto> ...\n", os.Args[0])
	flag.PrintDefaults()
}
<commit_msg>Write output files.<commit_after>package main

import (
	"bytes"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path"
	"strings"

	plugin "code.google.com/p/goprotobuf/compiler/plugin"
	"code.google.com/p/goprotobuf/proto"

	"github.com/dsymonds/gotoc/parser"
	"github.com/dsymonds/gotoc/resolver"
)

var (
	// Flags
	helpShort = flag.Bool("h", false, "Show usage text (same as --help).")
	helpLong  = flag.Bool("help", false, "Show usage text (same as -h).")

	importPath     = flag.String("import_path", ".", "Comma-separated list of paths to search for imports.")
	pluginBinary   = flag.String("plugin", "protoc-gen-go", "The code generator plugin to use.")
	descriptorOnly = flag.Bool("descriptor_only", false, "Whether to print out only the FileDescriptorSet.")
)

func fullPath(binary string, paths []string) string {
	if strings.Index(binary, "/") >= 0 {
		// path with path component
		return binary
	}
	for _, p := range paths {
		full := path.Join(p, binary)
		fi, err := os.Stat(full)
		if err == nil && !fi.IsDir() {
			return full
		}
	}
	return ""
}

func main() {
	flag.Usage = usage
	flag.Parse()
	if *helpShort || *helpLong || flag.NArg() == 0 {
		flag.Usage()
		os.Exit(1)
	}

	fds, err := parser.ParseFiles(flag.Args(), strings.Split(*importPath, ","))
	if err != nil {
		log.Fatalf("Failed parsing: %v", err)
	}
	if err := resolver.ResolveSymbols(fds); err != nil {
		log.Fatalf("Failed resolving symbols: %v", err)
	}

	if *descriptorOnly {
		proto.MarshalText(os.Stdout, fds)
		os.Exit(0)
	}

	//fmt.Println("-----")
	//proto.MarshalText(os.Stdout, fds)
	//fmt.Println("-----")

	// Prepare request.
	cgRequest := &plugin.CodeGeneratorRequest{
		FileToGenerate: flag.Args(),
		// TODO: proto_file should be topologically sorted (bottom-up)
		ProtoFile: fds.File,
	}
	buf, err := proto.Marshal(cgRequest)
	if err != nil {
		log.Fatalf("Failed marshaling CG request: %v", err)
	}

	// Find plugin.
	pluginPath := fullPath(*pluginBinary, strings.Split(os.Getenv("PATH"), ":"))
	if pluginPath == "" {
		log.Fatalf("Failed finding plugin binary %q", *pluginBinary)
	}

	// Run the plugin subprocess.
	cmd := &exec.Cmd{
		Path:   pluginPath,
		Stdin:  bytes.NewBuffer(buf),
		Stderr: os.Stderr,
	}
	buf, err = cmd.Output()
	if err != nil {
		log.Fatalf("Failed running plugin: %v", err)
	}

	// Parse the response.
	cgResponse := new(plugin.CodeGeneratorResponse)
	if err = proto.Unmarshal(buf, cgResponse); err != nil {
		log.Fatalf("Failed unmarshaling CG response: %v", err)
	}

	// TODO: check cgResponse.Error

	for _, f := range cgResponse.File {
		// TODO: If f.Name is nil, the content should be appended to the previous file.
		if f.Name == nil || f.Content == nil {
			log.Fatal("Malformed CG response")
		}
		if err := ioutil.WriteFile(*f.Name, []byte(*f.Content), 0644); err != nil {
			log.Fatalf("Failed writing output file: %v", err)
		}
	}
}

func usage() {
	fmt.Fprintf(os.Stderr, "Usage:  %s [options] <foo.proto> ...\n", os.Args[0])
	flag.PrintDefaults()
}

{'fraction_non_alphanumeric': 0.17529674866678135, 'fraction_numerical': 0.002408394976776191, 'mean_word_length': 3.585173501577287, 'pattern_counts': {'":': 2, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2378179', 'n_tokens_mistral': 2232, 'n_tokens_neox': 2131, 'n_words': 519}