<commit_before>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/scanner"
	"go/token"
	"io/ioutil"
	"os"
	pathutil "path"
	"strings"
)


var (
	// main operation modes
	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')")
	simplifyAST = flag.Bool("s", false, "simplify code")

	// debugging support
	comments = flag.Bool("comments", true, "print comments")
	trace    = flag.Bool("trace", false, "print parse trace")
	printAST = flag.Bool("ast", false, "print AST (before rewrites)")

	// layout control
	tabWidth  = flag.Int("tabwidth", 8, "tab width")
	tabIndent = flag.Bool("tabindent", true, "indent with tabs independent of -spaces")
	useSpaces = flag.Bool("spaces", true, "align with spaces instead of tabs")
)


var (
	fset        = token.NewFileSet()
	exitCode    = 0
	rewrite     func(*ast.File) *ast.File
	parserMode  uint
	printerMode uint
)


func report(err os.Error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}


func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}


func initParserMode() {
	parserMode = uint(0)
	if *comments {
		parserMode |= parser.ParseComments
	}
	if *trace {
		parserMode |= parser.Trace
	}
}


func initPrinterMode() {
	printerMode = uint(0)
	if *tabIndent {
		printerMode |= printer.TabIndent
	}
	if *useSpaces {
		printerMode |= printer.UseSpaces
	}
}


func isGoFile(f *os.FileInfo) bool {
	// ignore non-Go files
	return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && strings.HasSuffix(f.Name, ".go")
}


func processFile(f *os.File) os.Error {
	src, err := ioutil.ReadAll(f)
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(fset, f.Name(), src, parserMode)

	if err != nil {
		return err
	}

	if *printAST {
		ast.Print(file)
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	if *simplifyAST {
		simplify(file)
	}

	var res bytes.Buffer
	_, err = (&printer.Config{printerMode, *tabWidth, nil}).Fprint(&res, fset, file)
	if err != nil {
		return err
	}

	if bytes.Compare(src, res.Bytes()) != 0 {
		// formatting has changed
		if *list {
			fmt.Fprintln(os.Stdout, f.Name())
		}
		if *write {
			err = ioutil.WriteFile(f.Name(), res.Bytes(), 0)
			if err != nil {
				return err
			}
		}
	}

	if !*list && !*write {
		_, err = os.Stdout.Write(res.Bytes())
	}

	return err
}


func processFileByName(filename string) os.Error {
	file, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer file.Close()
	return processFile(file)
}


type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, f *os.FileInfo) bool {
	return true<commit_msg>gofmt: no need for lexical compare of src and res (optimization)<commit_after>// Copyright 2009 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package main

import (
	"bytes"
	"flag"
	"fmt"
	"go/ast"
	"go/parser"
	"go/printer"
	"go/scanner"
	"go/token"
	"io/ioutil"
	"os"
	pathutil "path"
	"strings"
)


var (
	// main operation modes
	list        = flag.Bool("l", false, "list files whose formatting differs from gofmt's")
	write       = flag.Bool("w", false, "write result to (source) file instead of stdout")
	rewriteRule = flag.String("r", "", "rewrite rule (e.g., 'α[β:len(α)] -> α[β:]')")
	simplifyAST = flag.Bool("s", false, "simplify code")

	// debugging support
	comments = flag.Bool("comments", true, "print comments")
	trace    = flag.Bool("trace", false, "print parse trace")
	printAST = flag.Bool("ast", false, "print AST (before rewrites)")

	// layout control
	tabWidth  = flag.Int("tabwidth", 8, "tab width")
	tabIndent = flag.Bool("tabindent", true, "indent with tabs independent of -spaces")
	useSpaces = flag.Bool("spaces", true, "align with spaces instead of tabs")
)


var (
	fset        = token.NewFileSet()
	exitCode    = 0
	rewrite     func(*ast.File) *ast.File
	parserMode  uint
	printerMode uint
)


func report(err os.Error) {
	scanner.PrintError(os.Stderr, err)
	exitCode = 2
}


func usage() {
	fmt.Fprintf(os.Stderr, "usage: gofmt [flags] [path ...]\n")
	flag.PrintDefaults()
	os.Exit(2)
}


func initParserMode() {
	parserMode = uint(0)
	if *comments {
		parserMode |= parser.ParseComments
	}
	if *trace {
		parserMode |= parser.Trace
	}
}


func initPrinterMode() {
	printerMode = uint(0)
	if *tabIndent {
		printerMode |= printer.TabIndent
	}
	if *useSpaces {
		printerMode |= printer.UseSpaces
	}
}


func isGoFile(f *os.FileInfo) bool {
	// ignore non-Go files
	return f.IsRegular() && !strings.HasPrefix(f.Name, ".") && strings.HasSuffix(f.Name, ".go")
}


func processFile(f *os.File) os.Error {
	src, err := ioutil.ReadAll(f)
	if err != nil {
		return err
	}

	file, err := parser.ParseFile(fset, f.Name(), src, parserMode)

	if err != nil {
		return err
	}

	if *printAST {
		ast.Print(file)
	}

	if rewrite != nil {
		file = rewrite(file)
	}

	if *simplifyAST {
		simplify(file)
	}

	var buf bytes.Buffer
	_, err = (&printer.Config{printerMode, *tabWidth, nil}).Fprint(&buf, fset, file)
	if err != nil {
		return err
	}
	res := buf.Bytes()

	if !bytes.Equal(src, res) {
		// formatting has changed
		if *list {
			fmt.Fprintln(os.Stdout, f.Name())
		}
		if *write {
			err = ioutil.WriteFile(f.Name(), res, 0)
			if err != nil {
				return err
			}
		}
	}

	if !*list && !*write {
		_, err = os.Stdout.Write(res)
	}

	return err
}


func processFileByName(filename string) os.Error {
	file, err := os.Open(filename, os.O_RDONLY, 0)
	if err != nil {
		return err
	}
	defer file.Close()
	return processFile(file)
}


type fileVisitor chan os.Error

func (v fileVisitor) VisitDir(path string, f *os.FileInfo) bool {
	return true
{'fraction_non_alphanumeric': 0.15227758592336788, 'fraction_numerical': 0.0041111659266568, 'mean_word_length': 3.4168482207697894, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '884246', 'n_tokens_mistral': 2419, 'n_tokens_neox': 2257, 'n_words': 610}