<commit_before>package main

import (
	"os"
	"path/filepath"
	"fmt"
	"regexp"
	"bytes"
	goopt "github.com/droundy/goopt"
	"./highlight"
	"./ignore"
)

var Author = "<NAME>"
var Version = "0.3"
var Summary = "gr [OPTS] string-to-search\n"

var byteNewLine []byte = []byte("\n")
// FIXME: global variable :(
// Used to prevent appear of sparse newline at the end of output
var prependNewLine = false


var onlyName = goopt.Flag([]string{"-n", "--filename"}, []string{},
	"print only filenames", "")
var ignoreFiles = goopt.Strings([]string{"-x", "--exclude"}, "RE",
	"exclude files that match the regexp from search")
var singleline = goopt.Flag([]string{"-s", "--singleline"}, []string{},
	"match on a single line (^/$ will be beginning/end of line)", "")
var replace = goopt.String([]string{"-r", "--replace"}, "",
	"replace found substrings with this string")
var force = goopt.Flag([]string{"--force"}, []string{},
	"force replacement in binary files", "")

func main() {
	goopt.Author = Author
	goopt.Version = Version
	goopt.Summary = Summary
	goopt.Parse(nil)

	cwd, _ := os.Getwd()
	ignorer := ignore.New(cwd)
	goopt.Summary += fmt.Sprintf("\n%s\n", ignorer)

	ignorer.Append(*ignoreFiles)

	if len(goopt.Args) == 0 {
		println(goopt.Usage())
		return
	}

	pattern, err := regexp.Compile(goopt.Args[0])
	errhandle(err, true, "can't compile regexp %s", goopt.Args[0])

	searchFiles(pattern, ignorer)
}

func errhandle(err os.Error, exit bool, moreinfo string, a ...interface{}) {
	if err == nil {
		return
	}
	fmt.Fprintf(os.Stderr, "ERR %s\n%s\n", err,
		fmt.Sprintf(moreinfo, a...))
	if exit {
		os.Exit(1)
	}
}

func searchFiles(pattern *regexp.Regexp, ignorer ignore.Ignorer) {
	v := &GRVisitor{pattern, ignorer}

	errors := make(chan os.Error, 64)

	filepath.Walk(".", v, errors)

	select {
	case err := <-errors:
		errhandle(err, true, "some error")
	default:
	}
}

type GRVisitor struct {
	pattern *regexp.Regexp
	ignorer ignore.Ignorer
}

func (v *GRVisitor) VisitDir(fn string, fi *os.FileInfo) bool {
	return !v.ignorer.Ignore(fi.Name, true)
}

func (v *GRVisitor) VisitFile(fn string, fi *os.FileInfo) {
	if !fi.IsRegular() {
		return
	}

	if fi.Size >= 1024*1024*10 {
		fmt.Fprintf(os.Stderr, "Skipping %s, too big: %d\n", fn, fi.Size)
		return
	}

	if fi.Size == 0 {
		return
	}

	if v.ignorer.Ignore(fn, false) {
		return
	}

	f, content := v.GetFileAndContent(fn, fi)
	defer f.Close()

	if len(*replace) == 0 {
		v.SearchFile(fn, content)
		return
	}

	changed, result := v.ReplaceInFile(fn, content)
	if changed {
		f.Seek(0, 0)
		n, err := f.Write(result)
		errhandle(err, true, "Error writing replacement in file %s", fn)
		if int64(n) > fi.Size {
			err := f.Truncate(int64(n))
			errhandle(err, true, "Error truncating file to size %d", f)
		}
	}
}

func (v *GRVisitor) GetFileAndContent(fn string, fi *os.FileInfo) (f *os.File, content []byte) {
	var err os.Error
	var msg string

	if len(*replace) > 0 {
		f, err = os.Open(fn, os.O_RDWR, 0666)
		msg = "can't open file %s for reading and writing"
	} else {
		f, err = os.Open(fn, os.O_RDONLY, 0666)
		msg = "can't open file %s for reading"
	}

	if err != nil {
		errhandle(err, false, msg, fn)
		return
	}

	content = make([]byte, fi.Size)
	n, err := f.Read(content)
	errhandle(err, true, "can't read file %s", fn)
	if int64(n) != fi.Size {
		panic(fmt.Sprintf("Not whole file was read, only %d from %d",
			n, fi.Size))<commit_msg>fix file truncation on replace<commit_after>package main

import (
	"os"
	"path/filepath"
	"fmt"
	"regexp"
	"bytes"
	goopt "github.com/droundy/goopt"
	"./highlight"
	"./ignore"
)

var Author = "<NAME>"
var Version = "0.3"
var Summary = "gr [OPTS] string-to-search\n"

var byteNewLine []byte = []byte("\n")
// FIXME: global variable :(
// Used to prevent appear of sparse newline at the end of output
var prependNewLine = false


var onlyName = goopt.Flag([]string{"-n", "--filename"}, []string{},
	"print only filenames", "")
var ignoreFiles = goopt.Strings([]string{"-x", "--exclude"}, "RE",
	"exclude files that match the regexp from search")
var singleline = goopt.Flag([]string{"-s", "--singleline"}, []string{},
	"match on a single line (^/$ will be beginning/end of line)", "")
var replace = goopt.String([]string{"-r", "--replace"}, "",
	"replace found substrings with this string")
var force = goopt.Flag([]string{"--force"}, []string{},
	"force replacement in binary files", "")

func main() {
	goopt.Author = Author
	goopt.Version = Version
	goopt.Summary = Summary
	goopt.Parse(nil)

	cwd, _ := os.Getwd()
	ignorer := ignore.New(cwd)
	goopt.Summary += fmt.Sprintf("\n%s\n", ignorer)

	ignorer.Append(*ignoreFiles)

	if len(goopt.Args) == 0 {
		println(goopt.Usage())
		return
	}

	pattern, err := regexp.Compile(goopt.Args[0])
	errhandle(err, true, "can't compile regexp %s", goopt.Args[0])

	searchFiles(pattern, ignorer)
}

func errhandle(err os.Error, exit bool, moreinfo string, a ...interface{}) {
	if err == nil {
		return
	}
	fmt.Fprintf(os.Stderr, "ERR %s\n%s\n", err,
		fmt.Sprintf(moreinfo, a...))
	if exit {
		os.Exit(1)
	}
}

func searchFiles(pattern *regexp.Regexp, ignorer ignore.Ignorer) {
	v := &GRVisitor{pattern, ignorer}

	errors := make(chan os.Error, 64)

	filepath.Walk(".", v, errors)

	select {
	case err := <-errors:
		errhandle(err, true, "some error")
	default:
	}
}

type GRVisitor struct {
	pattern *regexp.Regexp
	ignorer ignore.Ignorer
}

func (v *GRVisitor) VisitDir(fn string, fi *os.FileInfo) bool {
	return !v.ignorer.Ignore(fi.Name, true)
}

func (v *GRVisitor) VisitFile(fn string, fi *os.FileInfo) {
	if !fi.IsRegular() {
		return
	}

	if fi.Size >= 1024*1024*10 {
		fmt.Fprintf(os.Stderr, "Skipping %s, too big: %d\n", fn, fi.Size)
		return
	}

	if fi.Size == 0 {
		return
	}

	if v.ignorer.Ignore(fn, false) {
		return
	}

	f, content := v.GetFileAndContent(fn, fi)
	defer f.Close()

	if len(*replace) == 0 {
		v.SearchFile(fn, content)
		return
	}

	changed, result := v.ReplaceInFile(fn, content)
	if changed {
		f.Seek(0, 0)
		n, err := f.Write(result)
		errhandle(err, true, "Error writing replacement in file %s", fn)
		if int64(n) < fi.Size {
			err := f.Truncate(int64(n))
			errhandle(err, true, "Error truncating file to size %d", f)
		}
	}
}

func (v *GRVisitor) GetFileAndContent(fn string, fi *os.FileInfo) (f *os.File, content []byte) {
	var err os.Error
	var msg string

	if len(*replace) > 0 {
		f, err = os.Open(fn, os.O_RDWR, 0666)
		msg = "can't open file %s for reading and writing"
	} else {
		f, err = os.Open(fn, os.O_RDONLY, 0666)
		msg = "can't open file %s for reading"
	}

	if err != nil {
		errhandle(err, false, msg, fn)
		return
	}

	content = make([]byte, fi.Size)
	n, err := f.Read(content)
	errhandle(err, true, "can't read file %s", fn)
	if int64(n) != fi.Size {
		panic(fmt.Sprintf("Not whole file was read, only %d from %d",
			n, fi.Size))
{'fraction_non_alphanumeric': 0.17351865398683247, 'fraction_numerical': 0.010826627651792245, 'mean_word_length': 3.790469516468115, 'pattern_counts': {'":': 0, '<': 8, '<?xml version=': 0, '>': 10, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6382922', 'n_tokens_mistral': 2766, 'n_tokens_neox': 2598, 'n_words': 715}