<commit_before>package main

import (
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"
)

//The Slupr runner.

var runner = template.Must(template.New("main").Parse(`
package main

import (
  "log"

  "github.com/omeid/slurp/s"

  //client "{{ . }}"
)

func init() {
  log.Println("Starting...")
}

func main() {
  log.Println("Running....")

  slurp := s.NewBuild()
  
  Slurp(slurp)

  slurp.Run("default").Wait()

  log.Println("End.")
}`))

var (
	slurpfile = "slurp.go"
	cwd       string
	gopath    = os.Getenv("GOPATH")
)

func main() {

	log.Println("Start.")
	if gopath == "" {
		log.Fatal("$GOPATH must be set.")
	}

	err := run()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Done.")
}

func run() error {
	err, path := generate()
	if err != nil {
		return err
	}
	//Don't forget to clean up.
	//defer os.RemoveAll(path)

	cmd := exec.Command("go", "run",
		filepath.Join(path, "main.go"),
		filepath.Join(path, "slurp.go"),
	)

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()

	if err != nil {
		return err
	}

	return nil
}

func generate() (error, string) {

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset
	f, err := parser.ParseFile(fset, slurpfile, nil, parser.ParseComments)
	if err != nil {
		return err, ""
	}

	f.Name.Name = "main"               //Change package name
	f.Comments = []*ast.CommentGroup{} //Remove comments

	tmp, err := ioutil.TempDir(os.TempDir(), "slurp-")
	if err != nil {
		return err, ""
	}

	err = writeFileSet(filepath.Join(tmp, "slurp.go"), fset, f)
	if err != nil {
		return err, tmp
	}

	file, err := os.Create(filepath.Join(tmp, "main.go"))

	cwd, err = os.Getwd()
	if err != nil {
		return err, tmp
	}

	path, err := filepath.Rel(filepath.Join(gopath, "src"), cwd)
	if err != nil {
		return err, tmp
	}

	err = runner.Execute(file, path) //This should never fail, see MustParse.
	err = file.Close()

	if err != nil {
		return err, tmp
	}

	return nil, tmp

}

func writeFileSet(filepath string, fset *token.FileSet, node interface{}) error {
	// Print the modified AST.
	file, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	return format.Node(file, fset, node)
}<commit_msg>Copy the whole package, bom.<commit_after>package main

import (
	"errors"
	"fmt"
	"go/ast"
	"go/format"
	"go/parser"
	"go/token"
	"io/ioutil"
	"log"
	"os"
	"os/exec"
	"path/filepath"
	"text/template"
)

//The Slupr runner.

var runner = template.Must(template.New("main").Parse(`
package main

import (
  "log"

  "github.com/omeid/slurp/s"

  client "{{ . }}/tmp"
)

func init() {
  log.Println("Starting...")
}

func main() {
  log.Println("Running....")

  slurp := s.NewBuild()
  
  client.Slurp(slurp)

  slurp.Run("default").Wait()

  log.Println("End.")
}`))

var (
	slurpfile = "slurp.go"
	cwd       string
	gopath    = os.Getenv("GOPATH")
)

func main() {

	log.Println("Start.")
	if gopath == "" {
		log.Fatal("$GOPATH must be set.")
	}

	err := run()
	if err != nil {
		log.Fatal(err)
	}
	log.Println("Done.")
}

func run() error {
	path, err := generate()
	if err != nil {
		return err
	}
	//Don't forget to clean up.
	//defer os.RemoveAll(path)

	cmd := exec.Command("go", "run", filepath.Join(path, "main.go"))

	cmd.Stdin = os.Stdin
	cmd.Stdout = os.Stdout
	cmd.Stderr = os.Stderr

	err = cmd.Run()

	if err != nil {
		return err
	}

	return nil
}

func generate() (string, error) {

	//Let's grab a temp folder.
	path, err := ioutil.TempDir(filepath.Join(gopath, "src"), "slurp-run-")
	if err != nil {
		return "", err
	}

	tmp := filepath.Join(path, "tmp")
	err = os.Mkdir(tmp, 0700)
	if err != nil {
		return path, err
	}

	cwd, err = os.Getwd()
	if err != nil {
		return path, err
	}

	// Create the AST by parsing src.
	fset := token.NewFileSet() // positions are relative to fset

	fmt.Printf("cwd %+v\n", cwd)
	pkgs, err := parser.ParseDir(fset, cwd, nil, parser.ParseComments)
	if err != nil {
		return path, err
	}

	if len(pkgs) > 1 {
		return path, errors.New("Error: Multiple packages detected.")
	}

	for _, pkg := range pkgs {
		//This loop always runs once. I don't know of any other way to get the pkg out of pkgs
		// witout understanding the names.
		for name, f := range pkg.Files {
			f.Name.Name = "tmp" //Change package name
			if filepath.Base(name) == slurpfile {
				f.Comments = []*ast.CommentGroup{} //Remove comments
			}

			name, err = filepath.Rel(cwd, name)
			if err != nil {
				//Should never get error. But just incase.
				return path, err
			}
			err = writeFileSet(filepath.Join(tmp, name), fset, f)
			if err != nil {
				return path, err
			}
		}
	}

	file, err := os.Create(filepath.Join(path, "main.go"))

	tmp, err = filepath.Rel(filepath.Join(gopath, "src"), path)
	if err != nil {
		return path, err
	}

	err = runner.Execute(file, tmp) //This should never fail, see MustParse.
	err = file.Close()

	if err != nil {
		return path, err
	}

	return path, nil

}

func writeFileSet(filepath string, fset *token.FileSet, node interface{}) error {
	// Print the modified AST.
	file, err := os.Create(filepath)
	if err != nil {
		return err
	}
	defer file.Close()

	return format.Node(file, fset, node)
}
{'fraction_non_alphanumeric': 0.17099034273811778, 'fraction_numerical': 0.0009467903806097331, 'mean_word_length': 2.95952023988006, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1605407', 'n_tokens_mistral': 2219, 'n_tokens_neox': 2081, 'n_words': 540}