<commit_before>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var (
	regenerate = flag.Bool("regenerate", false, "regenerate files")

	protobufVersion = "3.7.0"
	golangVersions  = []string{"1.9.7", "1.10.8", "1.11.5", "1.12"}
	golangLatest    = golangVersions[len(golangVersions)-1]

	// List of directories to avoid auto-deleting. After updating the versions,
	// it is worth placing the previous versions here for some time since
	// other developers may still have working branches using the old versions.
	keepDirs = []string{"protobuf-3.6.1", "go1.10.7", "go1.11.4"}

	// Variables initialized by mustInitDeps.
	goPath     string
	modulePath string
)

func Test(t *testing.T) {
	mustInitDeps(t)

	if *regenerate {
		t.Run("Generate", func(t *testing.T) {
			fmt.Print(mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-types", "-execute"))
			fmt.Print(mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-protos", "-execute"))
			files := strings.Split(strings.TrimSpace(mustRunCommand(t, ".", "git", "ls-files", "*.go")), "\n")
			mustRunCommand(t, ".", append([]string{"gofmt", "-w"}, files...)...)
		})
		t.SkipNow()
	}

	for _, v := range golangVersions {
		t.Run("Go"+v, func(t *testing.T) {
			runGo := func(label, workDir string, args ...string) {
				args[0] += v
				t.Run(label, func(t *testing.T) {
					t.Parallel()
					mustRunCommand(t, workDir, args...)
				})
			}
			workDir := filepath.Join(goPath, "src", modulePath)
			runGo("Build", workDir, "go", "build", "./...")
			runGo("TestNormal", workDir, "go", "test", "-race", "./...")
			runGo("TestPureGo", workDir, "go", "test", "-race", "-tags", "purego", "./...")
			if v == golangLatest {
				runGo("TestProto1Legacy", workDir, "go", "test", "-race", "-tags", "proto1_legacy", "./...")
				runGo("TestProtocGenGo", "cmd/protoc-gen-go/testdata", "go", "test")
				runGo("TestProtocGenGoGRPC", "cmd/protoc-gen-go-grpc/testdata", "go", "test")
			}
		})
	}

	t.Run("GeneratedGoFiles", func(t *testing.T) {
		diff := mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-types")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("stale generated files:\n%v", diff)
		}
		diff = mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-protos")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("stale generated files:\n%v", diff)
		}
	})
	t.Run("FormattedGoFiles", func(t *testing.T) {
		files := strings.Split(strings.TrimSpace(mustRunCommand(t, ".", "git", "ls-files", "*.go")), "\n")
		diff := mustRunCommand(t, ".", append([]string{"gofmt", "-d"}, files...)...)
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("unformatted source files:\n%v", diff)
		}
	})
	t.Run("CommittedGitChanges", func(t *testing.T) {
		diff := mustRunCommand(t, ".", "git", "diff", "--no-prefix", "HEAD")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("uncommitted changes:\n%v", diff)
		}
	})
	t.Run("TrackedGitFiles", func(t *testing.T) {
		diff := mustRunCommand(t, ".", "git", "ls-files", "--others", "--exclude-standard")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("untracked files:\n%v", diff)
		}
	})
}

func mustInitDeps(t *testing.T) {
	check := func(err error) {
		t.Helper()
		if err != nil {
			t.Fatal(err)
		}
	}

	// Determine the directory to place the test directory.
	repoRoot, err := os.Getwd()
	check(err)
	testDir := filepath.Join(repoRoot, ".cache")
	check(os.MkdirAll(testDir, 0775))

	// Travis-CI has a hard-coded timeout where it kills the test after
	// 10 minutes of a lack of activity on stdout.
	// We work around this restriction by periodically printing the timestamp.
	ticker := time.NewTicker(5 * time.Minute)
	done := make(chan struct{})
	go func() {
		now := time.Now()
		for {
			select {
			case t := <-ticker.C:
				fmt.Printf("\tt=%0.fmin\n", t.Sub(now).Minutes())
			case <-done:
				return
			}
		}
	}()
	defer close(done)
	defer ticker.Stop()

	// Delete the current directory if non-empty,
	// which only occurs if a dependency failed to initialize properly.
	var workingDir string
	defer func() {
		if workingDir != "" {
			os.RemoveAll(workingDir) // best-effort
		}
	}()

	// Delete other sub-directories that are no longer relevant.
	defer func() {
		subDirs := map[string]bool{"bin": true, "gocache": true, "gopath": true}
		subDirs["protobuf-"+protobufVersion] = true
		for _, v := range golangVersions {
			subDirs["go"+v] = true
		}
		for _, d := range keepDirs {
			subDirs[d] = true
		}

		fis, _ := ioutil.ReadDir(testDir)
		for _, fi := range fis {
			if !subDirs[fi.Name()] {
				fmt.Printf("delete %v\n", fi.Name())
				os.RemoveAll(filepath.Join(testDir, fi.Name())) // best-effort
			}
		}
	}()

	// The bin directory contains symlinks to each tool by version.
	// It is safe to delete this directory and run the test script from scratch.
	binPath := filepath.Join(testDir, "bin")
	check(os.RemoveAll(binPath))
	check(os.Mkdir(binPath, 0775))
	check(os.Setenv("PATH", binPath+":"+os.Getenv("PATH")))
	registerBinary := func(name, path string) {
		check(os.Symlink(path, filepath.Join(binPath, name)))
	}

	// Download and build the protobuf toolchain.
	// We avoid downloading the pre-compiled binaries since they do not contain
	// the conformance test runner.
	workingDir = filepath.Join(testDir, "protobuf-"+protobufVersion)
	if _, err := os.Stat(workingDir); err != nil {
		fmt.Printf("download %v\n", filepath.Base(workingDir))<commit_msg>test.bash: purge unused directories based on timeout<commit_after>// Copyright 2019 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

// +build ignore

package main

import (
	"archive/tar"
	"bytes"
	"compress/gzip"
	"flag"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"regexp"
	"runtime"
	"strings"
	"testing"
	"time"
)

var (
	regenerate = flag.Bool("regenerate", false, "regenerate files")

	protobufVersion = "3.7.0"
	golangVersions  = []string{"1.9.7", "1.10.8", "1.11.5", "1.12"}
	golangLatest    = golangVersions[len(golangVersions)-1]

	// purgeTimeout determines the maximum age of unused sub-directories.
	purgeTimeout = 30 * 24 * time.Hour // 1 month

	// Variables initialized by mustInitDeps.
	goPath     string
	modulePath string
)

func Test(t *testing.T) {
	mustInitDeps(t)

	if *regenerate {
		t.Run("Generate", func(t *testing.T) {
			fmt.Print(mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-types", "-execute"))
			fmt.Print(mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-protos", "-execute"))
			files := strings.Split(strings.TrimSpace(mustRunCommand(t, ".", "git", "ls-files", "*.go")), "\n")
			mustRunCommand(t, ".", append([]string{"gofmt", "-w"}, files...)...)
		})
		t.SkipNow()
	}

	for _, v := range golangVersions {
		t.Run("Go"+v, func(t *testing.T) {
			runGo := func(label, workDir string, args ...string) {
				args[0] += v
				t.Run(label, func(t *testing.T) {
					t.Parallel()
					mustRunCommand(t, workDir, args...)
				})
			}
			workDir := filepath.Join(goPath, "src", modulePath)
			runGo("Build", workDir, "go", "build", "./...")
			runGo("TestNormal", workDir, "go", "test", "-race", "./...")
			runGo("TestPureGo", workDir, "go", "test", "-race", "-tags", "purego", "./...")
			if v == golangLatest {
				runGo("TestProto1Legacy", workDir, "go", "test", "-race", "-tags", "proto1_legacy", "./...")
				runGo("TestProtocGenGo", "cmd/protoc-gen-go/testdata", "go", "test")
				runGo("TestProtocGenGoGRPC", "cmd/protoc-gen-go-grpc/testdata", "go", "test")
			}
		})
	}

	t.Run("GeneratedGoFiles", func(t *testing.T) {
		diff := mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-types")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("stale generated files:\n%v", diff)
		}
		diff = mustRunCommand(t, ".", "go", "run", "./internal/cmd/generate-protos")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("stale generated files:\n%v", diff)
		}
	})
	t.Run("FormattedGoFiles", func(t *testing.T) {
		files := strings.Split(strings.TrimSpace(mustRunCommand(t, ".", "git", "ls-files", "*.go")), "\n")
		diff := mustRunCommand(t, ".", append([]string{"gofmt", "-d"}, files...)...)
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("unformatted source files:\n%v", diff)
		}
	})
	t.Run("CommittedGitChanges", func(t *testing.T) {
		diff := mustRunCommand(t, ".", "git", "diff", "--no-prefix", "HEAD")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("uncommitted changes:\n%v", diff)
		}
	})
	t.Run("TrackedGitFiles", func(t *testing.T) {
		diff := mustRunCommand(t, ".", "git", "ls-files", "--others", "--exclude-standard")
		if strings.TrimSpace(diff) != "" {
			t.Fatalf("untracked files:\n%v", diff)
		}
	})
}

func mustInitDeps(t *testing.T) {
	check := func(err error) {
		t.Helper()
		if err != nil {
			t.Fatal(err)
		}
	}

	// Determine the directory to place the test directory.
	repoRoot, err := os.Getwd()
	check(err)
	testDir := filepath.Join(repoRoot, ".cache")
	check(os.MkdirAll(testDir, 0775))

	// Travis-CI has a hard-coded timeout where it kills the test after
	// 10 minutes of a lack of activity on stdout.
	// We work around this restriction by periodically printing the timestamp.
	ticker := time.NewTicker(5 * time.Minute)
	done := make(chan struct{})
	go func() {
		now := time.Now()
		for {
			select {
			case t := <-ticker.C:
				fmt.Printf("\tt=%0.fmin\n", t.Sub(now).Minutes())
			case <-done:
				return
			}
		}
	}()
	defer close(done)
	defer ticker.Stop()

	// Delete the current directory if non-empty,
	// which only occurs if a dependency failed to initialize properly.
	var workingDir string
	defer func() {
		if workingDir != "" {
			os.RemoveAll(workingDir) // best-effort
		}
	}()

	// Delete other sub-directories that are no longer relevant.
	defer func() {
		subDirs := map[string]bool{"bin": true, "gocache": true, "gopath": true}
		subDirs["protobuf-"+protobufVersion] = true
		for _, v := range golangVersions {
			subDirs["go"+v] = true
		}

		now := time.Now()
		fis, _ := ioutil.ReadDir(testDir)
		for _, fi := range fis {
			if subDirs[fi.Name()] {
				os.Chtimes(filepath.Join(testDir, fi.Name()), now, now) // best-effort
				continue
			}
			if now.Sub(fi.ModTime()) < purgeTimeout {
				continue
			}
			fmt.Printf("delete %v\n", fi.Name())
			os.RemoveAll(filepath.Join(testDir, fi.Name())) // best-effort
		}
	}()

	// The bin directory contains symlinks to each tool by version.
	// It is safe to delete this directory and run the test script from scratch.
	binPath := filepath.Join(testDir, "bin")
	check(os.RemoveAll(binPath))
	check(os.Mkdir(binPath, 0775))
	check(os.Setenv("PATH", binPath+":"+os.Getenv("PATH")))
	registerBinary := func(name, path string) {
		check(os.Symlink(path, filepath.Join(binPath, name)))
	}

	// Download and build the protobuf toolchain.
	// We avoid downloading the pre-compiled binaries since they do not contain
	// the conformance test runner.
	workingDir = filepath.Join(testDir, "protobuf-"+protobufVersion)
	if _, err := os.Stat(workingDir); err != nil {
		fmt.Printf("download %v\n", filepath.Base(workingDir))
{'fraction_non_alphanumeric': 0.19245249978110499, 'fraction_numerical': 0.007880220646178092, 'mean_word_length': 3.5963782696177065, 'pattern_counts': {'":': 8, '<': 8, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 6, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1957727', 'n_tokens_mistral': 4539, 'n_tokens_neox': 4151, 'n_words': 978}