<commit_before>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License.  You may obtain a copy
// of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
// License for the specific language governing permissions and limitations
// under the License.

package utils

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"testing"
	"time"

	"bazil.org/fuse"
	"golang.org/x/sys/unix"
)

const (
	// Maximum amount of time to wait for sandboxfs to come up and start serving.
	startupDeadlineSeconds = 10

	// Maximum amount of time to wait for sandboxfs to gracefully exit after an unmount.
	shutdownDeadlineSeconds = 5
)

// runState holds runtime information for an in-progress sandboxfs execution.
type runState struct {
	cmd *exec.Cmd
	out bytes.Buffer
	err bytes.Buffer
}

// run starts a background process to run sandboxfs and passes it the given arguments.
func run(arg ...string) (*runState, error) {
	bin := GetConfig().SandboxfsBinary

	var state runState
	state.cmd = exec.Command(bin, arg...)
	state.cmd.Stdout = &state.out
	state.cmd.Stderr = &state.err
	if err := state.cmd.Start(); err != nil {
		return nil, fmt.Errorf("failed to start %s with arguments %v: %v", bin, arg, err)
	}
	return &state, nil
}

// wait awaits for completion of the process started by run and checks its exit status.
// Returns the textual contents of stdout and stderr for further inspection.
func wait(state *runState, wantExitStatus int) (string, string, error) {
	err := state.cmd.Wait()
	if wantExitStatus == 0 {
		if err != nil {
			return state.out.String(), state.err.String(), fmt.Errorf("got %v; want sandboxfs to exit with status 0", err)
		}
	} else {
		if err == nil {
			return state.out.String(), state.err.String(), fmt.Errorf("got 0; want sandboxfs to exit with status %d", wantExitStatus)
		}
		status := err.(*exec.ExitError).ProcessState.Sys().(syscall.WaitStatus)
		if wantExitStatus != status.ExitStatus() {
			return state.out.String(), state.err.String(), fmt.Errorf("got %v; want sandboxfs to exit with status %d", status.ExitStatus(), wantExitStatus)
		}
	}
	return state.out.String(), state.err.String(), nil
}

// RunAndWait invokes sandboxfs with the given arguments and waits for termination.
//
// TODO(jmmv): We should extend this function to also take what the expectations are on stdout and
// stderr to remove a lot of boilerplate from the tests... but we should probably wait until Go's
// 1.9 t.Helper() feature is available so that we can actually report failures/errors from here.
func RunAndWait(wantExitStatus int, arg ...string) (string, string, error) {
	state, err := run(arg...)
	if err != nil {
		return "", "", err
	}
	return wait(state, wantExitStatus)
}

// retry runs the given action until either it succeeds or the given deadline expires.  If the
// deadline expires, returns the last encountered error from the action.  Prints the given message
// when an error is encountered.
func retry(action func() error, message string, deadlineSeconds int) error {
	var lastErr error
	for tries := 0; tries < deadlineSeconds*10; tries++ {
		lastErr = action()
		if lastErr == nil {
			return nil
		}
		fmt.Fprintf(os.Stderr, "In retry attempt %d: %s: %v\n", tries, message, lastErr)
		time.Sleep(100 * time.Millisecond)
	}
	return lastErr<commit_msg>Silence eager retry warnings (#20)<commit_after>// Copyright 2017 Google Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License"); you may not
// use this file except in compliance with the License.  You may obtain a copy
// of the License at:
//
//     http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.  See the
// License for the specific language governing permissions and limitations
// under the License.

package utils

import (
	"bytes"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
	"syscall"
	"testing"
	"time"

	"bazil.org/fuse"
	"golang.org/x/sys/unix"
)

const (
	// Maximum amount of time to wait for sandboxfs to come up and start serving.
	startupDeadlineSeconds = 10

	// Maximum amount of time to wait for sandboxfs to gracefully exit after an unmount.
	shutdownDeadlineSeconds = 5
)

// runState holds runtime information for an in-progress sandboxfs execution.
type runState struct {
	cmd *exec.Cmd
	out bytes.Buffer
	err bytes.Buffer
}

// run starts a background process to run sandboxfs and passes it the given arguments.
func run(arg ...string) (*runState, error) {
	bin := GetConfig().SandboxfsBinary

	var state runState
	state.cmd = exec.Command(bin, arg...)
	state.cmd.Stdout = &state.out
	state.cmd.Stderr = &state.err
	if err := state.cmd.Start(); err != nil {
		return nil, fmt.Errorf("failed to start %s with arguments %v: %v", bin, arg, err)
	}
	return &state, nil
}

// wait awaits for completion of the process started by run and checks its exit status.
// Returns the textual contents of stdout and stderr for further inspection.
func wait(state *runState, wantExitStatus int) (string, string, error) {
	err := state.cmd.Wait()
	if wantExitStatus == 0 {
		if err != nil {
			return state.out.String(), state.err.String(), fmt.Errorf("got %v; want sandboxfs to exit with status 0", err)
		}
	} else {
		if err == nil {
			return state.out.String(), state.err.String(), fmt.Errorf("got 0; want sandboxfs to exit with status %d", wantExitStatus)
		}
		status := err.(*exec.ExitError).ProcessState.Sys().(syscall.WaitStatus)
		if wantExitStatus != status.ExitStatus() {
			return state.out.String(), state.err.String(), fmt.Errorf("got %v; want sandboxfs to exit with status %d", status.ExitStatus(), wantExitStatus)
		}
	}
	return state.out.String(), state.err.String(), nil
}

// RunAndWait invokes sandboxfs with the given arguments and waits for termination.
//
// TODO(jmmv): We should extend this function to also take what the expectations are on stdout and
// stderr to remove a lot of boilerplate from the tests... but we should probably wait until Go's
// 1.9 t.Helper() feature is available so that we can actually report failures/errors from here.
func RunAndWait(wantExitStatus int, arg ...string) (string, string, error) {
	state, err := run(arg...)
	if err != nil {
		return "", "", err
	}
	return wait(state, wantExitStatus)
}

// retry runs the given action until either it succeeds or the given deadline expires.  If the
// deadline expires, returns the last encountered error from the action.  Prints the given message
// when an error is encountered.
func retry(action func() error, message string, deadlineSeconds int) error {
	var lastErr error
	for tries := 0; tries < deadlineSeconds*10; tries++ {
		lastErr = action()
		if lastErr == nil {
			return nil
		}
		if tries > 10 {
			// Avoid cluttering the logs for the first few retries.  It's normal for
			// the first attempt to not succeed, which results in warning messages
			// printed for every test when there is nothing really wrong.
			fmt.Fprintf(os.Stderr, "In retry attempt %d: %s: %v\n", tries, message, lastErr)
		}
		time.Sleep(100 * time.Millisecond)
	}
	return lastErr
{'fraction_non_alphanumeric': 0.1108926464805348, 'fraction_numerical': 0.006291781360597719, 'mean_word_length': 3.6695226438188495, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7117407', 'n_tokens_mistral': 2412, 'n_tokens_neox': 2275, 'n_words': 935}