<commit_before>/*
Package test161 implements a library for testing OS/161 kernels. We use expect
to drive the sys161 system simulator and collect useful output using the stat
socket.
*/
package test161

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/kr/pty"
	"github.com/ops-class/test161/expect"
	"github.com/termie/go-shutil"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"regexp"
	"sync"
	"time"
)

const KERNEL_PROMPT = `OS/161 kernel [? for menu]: `
const SHELL_PROMPT = `OS/161$ `
const PROMPT_PATTERN = `(OS/161 kernel \[\? for menu\]\:|OS/161\$)\s$`
const BOOT = -1

type Test struct {

	// Input

	// Metadata
	Name        string   `yaml:"name" json:"name"`
	Description string   `yaml:"description" json:"description"`
	Tags        []string `yaml:"tags" json:"tags"`
	Depends     []string `yaml:"depends" json:"depends"`

	// Configuration chunks
	Sys161  Sys161Conf  `yaml:"sys161" json:"sys161"`
	Stat    StatConf    `yaml:"stat" json:"stat"`
	Monitor MonitorConf `yaml:"monitor" json:"monitor"`
	Misc    MiscConf    `yaml:"misc" json:"misc"`

	// Actual test commands to run
	Content string `fm:"content" yaml:"-" json:"-"`

	// Big lock that protects most fields shared between Run and getStats
	L *sync.Mutex `json:"-"`

	// Output

	ConfString string         `json:"confstring"` // Only set during once
	WallTime   TimeFixedPoint `json:"walltime"`   // Protected by L
	SimTime    TimeFixedPoint `json:"simtime"`    // Protected by L
	Commands   []Command      `json:"commands"`   // Protected by L
	Status     []Status       `json:"status"`     // Protected by L

	// Unproctected Private fields
	tempDir     string // Only set once
	startTime   int64  // Only set once
	statStarted bool   // Only changed once

	sys161         *expect.Expect // Protected by L
	progressTime   float64        // Protected by L
	currentCommand *Command       // Protected by L
	commandCounter uint           // Protected by L
	currentOutput  OutputLine     // Protected by L

	// Fields used by getStats but shared with Run
	statCond    *sync.Cond // Used by the main loop to wait for stat reception
	statRecord  bool       // Protected by statCond.L
	statMonitor bool       // Protected by statCond.L

	// Output channels
	statChan chan Stat // Nonblocking write
}

type Command struct {
	Type         string       `json:"type"`
	Input        InputLine    `json:"input"`
	Output       []OutputLine `json:"output"`
	SummaryStats Stat         `json:"summarystats"`
	AllStats     []Stat       `json:"stats"`
	Monitored    bool         `json:"monitored"`
}

type InputLine struct {
	WallTime TimeFixedPoint `json:"walltime"`
	SimTime  TimeFixedPoint `json:"simtime"`
	Line     string         `json:"line"`
}

type OutputLine struct {
	WallTime TimeFixedPoint `json:"walltime"`
	SimTime  TimeFixedPoint `json:"simtime"`
	Buffer   bytes.Buffer   `json:"-"`
	Line     string         `json:"line"`
}

type Status struct {
	WallTime TimeFixedPoint `json:"walltime"`
	SimTime  TimeFixedPoint `json:"simtime"`
	Status   string         `json:"status"`
	Message  string         `json:"message"`
}

type TimeFixedPoint float64

// MarshalJSON prints our TimeFixedPoint type as a fixed point float for JSON.
func (t TimeFixedPoint) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("%.6f", t)), nil
}

// getTimeFixedPoint returns the current wall clock time as a TimeFixedPoint
func (t *Test) getWallTime() TimeFixedPoint {
	return TimeFixedPoint(float64(time.Now().UnixNano()-t.startTime) / float64(1000*1000*1000))
}

// Run a test161 test.
func (t *Test) Run(root string) (err error) {

	// Merge in test161 defaults for any missing configuration values
	err = t.MergeConf(CONF_DEFAULTS)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}

	// Create temp directory.
	tempRoot, err := ioutil.TempDir(t.Misc.TempDir, "test161")
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	defer os.RemoveAll(tempRoot)
	t.tempDir = path.Join(tempRoot, "root")

	// Copy root.
	err = shutil.CopyTree(root, t.tempDir, nil)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}

	// Make sure we have a kernel.
	kernelTarget := path.Join(t.tempDir, "kernel")
	_, err = os.Stat(kernelTarget)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}

	// Generate an alternate configuration to prevent collisions.
	confTarget := path.Join(t.tempDir, "test161.conf")
	t.ConfString, err = t.PrintConf()
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	err = ioutil.WriteFile(confTarget, []byte(t.ConfString), 0440)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	if _, err := os.Stat(confTarget); os.IsNotExist(err) {
		t.addStatus("aborted", "")
		return err
	}

	// Create disks.
	if t.Sys161.Disk1.Enabled == "true" {
		create := exec.Command("disk161", "create", "LHD0.img", t.Sys161.Disk1.Bytes)
		create.Dir = t.tempDir
		err = create.Run()
		if err != nil {
			t.addStatus("aborted", "")
			return err
		}
	}
	if t.Sys161.Disk2.Enabled == "true" {
		create := exec.Command("disk161", "create", "LHD1.img", t.Sys161.Disk2.Bytes)
		create.Dir = t.tempDir
		err = create.Run()
		if err != nil {
			t.addStatus("aborted", "")
			return err
		}
	}

	// Serialize the current command state.
	t.L = &sync.Mutex{}

	// Coordinated with the getStat goroutine. I don't think that a channel
	// would work here.
	t.statCond = &sync.Cond{L: &sync.Mutex{}}

	// Initialize stat channel. Closed by getStats
	t.statChan = make(chan Stat)

	// Record stats during boot, but don't activate the monitor.
	t.statRecord = true
	t.statMonitor = false

	// Wait for either kernel or user prompts.
	prompts := regexp.MustCompile(PROMPT_PATTERN)

	// Set up the current command to point at boot
	t.commandCounter = 0
	t.currentCommand = &t.Commands[t.commandCounter]

	// Start sys161 and defer close.
	err = t.start161()
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	defer t.stop161()
	t.addStatus("started", "")

	for int(t.commandCounter) < len(t.Commands) {
		if t.commandCounter != 0 {
			err = t.sendCommand(t.currentCommand.Input.Line + "\n")
			if err != nil {
				t.addStatus("expect", "couldn't send a command")
				return nil
			}
			t.enableStats()
		}

		if int(t.commandCounter) == len(t.Commands)-1 {
			t.sys161.ExpectEOF()
			t.addStatus("shutdown", "")
			break
		}
		_, err := t.sys161.ExpectRegexp(prompts)
		t.disableStats()

		// Handle timeouts, unexpected shutdowns, and other errors
		if err == expect.ErrTimeout {
			t.addStatus("timeout", fmt.Sprintf("no prompt for %v s", t.Misc.PromptTimeout))
			return nil
		} else if err == io.EOF {
			t.addStatus("crash", "")
			return nil
		} else if err != nil {
			t.addStatus("expect", "")
			return nil
		}

		// Rotate running command to the next command, saving any previous
		// output as needed.
		t.L.Lock()
		if t.currentOutput.WallTime != 0.0 {
			t.currentOutput.Line = t.currentOutput.Buffer.String()
			t.currentCommand.Output = append(t.currentCommand.Output, t.currentOutput)
		}
		t.currentOutput = OutputLine{}
		t.commandCounter++
		t.currentCommand = &t.Commands[t.commandCounter]
		t.L.Unlock()
	}
	return nil
}

// sendCommand sends a command persistently. All the retry logic to deal with
// dropped characters is now here.
func (t *Test) sendCommand(commandLine string) error {

	// Temporarily lower the expect timeout.
	t.sys161.SetTimeout(time.Duration(t.Misc.CharacterTimeout) * time.Millisecond)
	defer t.sys161.SetTimeout(time.Duration(t.Misc.PromptTimeout) * time.Second)

	for _, character := range commandLine {
		retryCount := uint(0)
		for ; retryCount < t.Misc.CommandRetries; retryCount++ {
			err := t.sys161.Send(string(character))
			if err != nil {
				return err
			}
			_, err = t.sys161.ExpectRegexp(regexp.MustCompile(regexp.QuoteMeta(string(character))))
			if err == nil {
				break
			} else if err == expect.ErrTimeout {
				continue
			} else {
				return err
			}
		}
		if retryCount == t.Misc.CommandRetries {
			return errors.New("test161: timeout sending command")
		}
	}

	return nil
}

// start161 is a private helper function to start the sys161 expect process.
func (t *Test) start161() error {
	// Disable debugger connections on panic and set our alternate
	// configuration.
	run := exec.Command("sys161", "-X", "-c", "test161.conf", "kernel")
	run.Dir = t.tempDir
	pty, err := pty.Start(run)
	if err != nil {
		return err
	}

	// Get serious about killing things.
	killer := func() {
		run.Process.Signal(os.Kill)
	}

	// Set timeout at create to avoid hanging with early failures.
	t.sys161 = expect.Create(pty, killer, t, time.Duration(t.Misc.PromptTimeout)*time.Second)
	t.startTime = time.Now().UnixNano()

	return nil
}

func (t *Test) stop161() {
	t.Commands = t.Commands[0 : t.commandCounter+1]
	t.WallTime = t.getWallTime()
	t.sys161.Close()
}

func (t *Test) addStatus(status string, message string) {
	t.L.Lock()
	t.Status = append(t.Status, Status{
		WallTime: t.getWallTime(),
		SimTime:  t.SimTime,
		Status:   status,
		Message:  message,
	})
	t.L.Unlock()
}
<commit_msg>Small fix.<commit_after>/*
Package test161 implements a library for testing OS/161 kernels. We use expect
to drive the sys161 system simulator and collect useful output using the stat
socket.
*/
package test161

import (
	"bytes"
	"errors"
	"fmt"
	"github.com/kr/pty"
	"github.com/ops-class/test161/expect"
	"github.com/termie/go-shutil"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"regexp"
	"sync"
	"time"
)

const KERNEL_PROMPT = `OS/161 kernel [? for menu]: `
const SHELL_PROMPT = `OS/161$ `
const PROMPT_PATTERN = `(OS/161 kernel \[\? for menu\]\:|OS/161\$)\s$`
const BOOT = -1

type Test struct {

	// Input

	// Metadata
	Name        string   `yaml:"name" json:"name"`
	Description string   `yaml:"description" json:"description"`
	Tags        []string `yaml:"tags" json:"tags"`
	Depends     []string `yaml:"depends" json:"depends"`

	// Configuration chunks
	Sys161  Sys161Conf  `yaml:"sys161" json:"sys161"`
	Stat    StatConf    `yaml:"stat" json:"stat"`
	Monitor MonitorConf `yaml:"monitor" json:"monitor"`
	Misc    MiscConf    `yaml:"misc" json:"misc"`

	// Actual test commands to run
	Content string `fm:"content" yaml:"-" json:"-"`

	// Big lock that protects most fields shared between Run and getStats
	L *sync.Mutex `json:"-"`

	// Output

	ConfString string         `json:"confstring"` // Only set during once
	WallTime   TimeFixedPoint `json:"walltime"`   // Protected by L
	SimTime    TimeFixedPoint `json:"simtime"`    // Protected by L
	Commands   []Command      `json:"commands"`   // Protected by L
	Status     []Status       `json:"status"`     // Protected by L

	// Unproctected Private fields
	tempDir     string // Only set once
	startTime   int64  // Only set once
	statStarted bool   // Only changed once

	sys161         *expect.Expect // Protected by L
	progressTime   float64        // Protected by L
	currentCommand *Command       // Protected by L
	commandCounter uint           // Protected by L
	currentOutput  OutputLine     // Protected by L

	// Fields used by getStats but shared with Run
	statCond    *sync.Cond // Used by the main loop to wait for stat reception
	statRecord  bool       // Protected by statCond.L
	statMonitor bool       // Protected by statCond.L

	// Output channels
	statChan chan Stat // Nonblocking write
}

type Command struct {
	Type         string       `json:"type"`
	Input        InputLine    `json:"input"`
	Output       []OutputLine `json:"output"`
	SummaryStats Stat         `json:"summarystats"`
	AllStats     []Stat       `json:"stats"`
	Monitored    bool         `json:"monitored"`
}

type InputLine struct {
	WallTime TimeFixedPoint `json:"walltime"`
	SimTime  TimeFixedPoint `json:"simtime"`
	Line     string         `json:"line"`
}

type OutputLine struct {
	WallTime TimeFixedPoint `json:"walltime"`
	SimTime  TimeFixedPoint `json:"simtime"`
	Buffer   bytes.Buffer   `json:"-"`
	Line     string         `json:"line"`
}

type Status struct {
	WallTime TimeFixedPoint `json:"walltime"`
	SimTime  TimeFixedPoint `json:"simtime"`
	Status   string         `json:"status"`
	Message  string         `json:"message"`
}

type TimeFixedPoint float64

// MarshalJSON prints our TimeFixedPoint type as a fixed point float for JSON.
func (t TimeFixedPoint) MarshalJSON() ([]byte, error) {
	return []byte(fmt.Sprintf("%.6f", t)), nil
}

// getTimeFixedPoint returns the current wall clock time as a TimeFixedPoint
func (t *Test) getWallTime() TimeFixedPoint {
	return TimeFixedPoint(float64(time.Now().UnixNano()-t.startTime) / float64(1000*1000*1000))
}

// Run a test161 test.
func (t *Test) Run(root string) (err error) {
	// Serialize the current command state.
	t.L = &sync.Mutex{}

	// Merge in test161 defaults for any missing configuration values
	err = t.MergeConf(CONF_DEFAULTS)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}

	// Create temp directory.
	tempRoot, err := ioutil.TempDir(t.Misc.TempDir, "test161")
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	defer os.RemoveAll(tempRoot)
	t.tempDir = path.Join(tempRoot, "root")

	// Copy root.
	err = shutil.CopyTree(root, t.tempDir, nil)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}

	// Make sure we have a kernel.
	kernelTarget := path.Join(t.tempDir, "kernel")
	_, err = os.Stat(kernelTarget)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}

	// Generate an alternate configuration to prevent collisions.
	confTarget := path.Join(t.tempDir, "test161.conf")
	t.ConfString, err = t.PrintConf()
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	err = ioutil.WriteFile(confTarget, []byte(t.ConfString), 0440)
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	if _, err := os.Stat(confTarget); os.IsNotExist(err) {
		t.addStatus("aborted", "")
		return err
	}

	// Create disks.
	if t.Sys161.Disk1.Enabled == "true" {
		create := exec.Command("disk161", "create", "LHD0.img", t.Sys161.Disk1.Bytes)
		create.Dir = t.tempDir
		err = create.Run()
		if err != nil {
			t.addStatus("aborted", "")
			return err
		}
	}
	if t.Sys161.Disk2.Enabled == "true" {
		create := exec.Command("disk161", "create", "LHD1.img", t.Sys161.Disk2.Bytes)
		create.Dir = t.tempDir
		err = create.Run()
		if err != nil {
			t.addStatus("aborted", "")
			return err
		}
	}

	// Coordinated with the getStat goroutine. I don't think that a channel
	// would work here.
	t.statCond = &sync.Cond{L: &sync.Mutex{}}

	// Initialize stat channel. Closed by getStats
	t.statChan = make(chan Stat)

	// Record stats during boot, but don't activate the monitor.
	t.statRecord = true
	t.statMonitor = false

	// Wait for either kernel or user prompts.
	prompts := regexp.MustCompile(PROMPT_PATTERN)

	// Set up the current command to point at boot
	t.commandCounter = 0
	t.currentCommand = &t.Commands[t.commandCounter]

	// Start sys161 and defer close.
	err = t.start161()
	if err != nil {
		t.addStatus("aborted", "")
		return err
	}
	defer t.stop161()
	t.addStatus("started", "")

	for int(t.commandCounter) < len(t.Commands) {
		if t.commandCounter != 0 {
			err = t.sendCommand(t.currentCommand.Input.Line + "\n")
			if err != nil {
				t.addStatus("expect", "couldn't send a command")
				return nil
			}
			t.enableStats()
		}

		if int(t.commandCounter) == len(t.Commands)-1 {
			t.sys161.ExpectEOF()
			t.addStatus("shutdown", "")
			break
		}
		_, err := t.sys161.ExpectRegexp(prompts)
		t.disableStats()

		// Handle timeouts, unexpected shutdowns, and other errors
		if err == expect.ErrTimeout {
			t.addStatus("timeout", fmt.Sprintf("no prompt for %v s", t.Misc.PromptTimeout))
			return nil
		} else if err == io.EOF {
			t.addStatus("crash", "")
			return nil
		} else if err != nil {
			t.addStatus("expect", "")
			return nil
		}

		// Rotate running command to the next command, saving any previous
		// output as needed.
		t.L.Lock()
		if t.currentOutput.WallTime != 0.0 {
			t.currentOutput.Line = t.currentOutput.Buffer.String()
			t.currentCommand.Output = append(t.currentCommand.Output, t.currentOutput)
		}
		t.currentOutput = OutputLine{}
		t.commandCounter++
		t.currentCommand = &t.Commands[t.commandCounter]
		t.L.Unlock()
	}
	return nil
}

// sendCommand sends a command persistently. All the retry logic to deal with
// dropped characters is now here.
func (t *Test) sendCommand(commandLine string) error {

	// Temporarily lower the expect timeout.
	t.sys161.SetTimeout(time.Duration(t.Misc.CharacterTimeout) * time.Millisecond)
	defer t.sys161.SetTimeout(time.Duration(t.Misc.PromptTimeout) * time.Second)

	for _, character := range commandLine {
		retryCount := uint(0)
		for ; retryCount < t.Misc.CommandRetries; retryCount++ {
			err := t.sys161.Send(string(character))
			if err != nil {
				return err
			}
			_, err = t.sys161.ExpectRegexp(regexp.MustCompile(regexp.QuoteMeta(string(character))))
			if err == nil {
				break
			} else if err == expect.ErrTimeout {
				continue
			} else {
				return err
			}
		}
		if retryCount == t.Misc.CommandRetries {
			return errors.New("test161: timeout sending command")
		}
	}

	return nil
}

// start161 is a private helper function to start the sys161 expect process.
func (t *Test) start161() error {
	// Disable debugger connections on panic and set our alternate
	// configuration.
	run := exec.Command("sys161", "-X", "-c", "test161.conf", "kernel")
	run.Dir = t.tempDir
	pty, err := pty.Start(run)
	if err != nil {
		return err
	}

	// Get serious about killing things.
	killer := func() {
		run.Process.Signal(os.Kill)
	}

	// Set timeout at create to avoid hanging with early failures.
	t.sys161 = expect.Create(pty, killer, t, time.Duration(t.Misc.PromptTimeout)*time.Second)
	t.startTime = time.Now().UnixNano()

	return nil
}

func (t *Test) stop161() {
	t.Commands = t.Commands[0 : t.commandCounter+1]
	t.WallTime = t.getWallTime()
	t.sys161.Close()
}

func (t *Test) addStatus(status string, message string) {
	t.L.Lock()
	t.Status = append(t.Status, Status{
		WallTime: t.getWallTime(),
		SimTime:  t.SimTime,
		Status:   status,
		Message:  message,
	})
	t.L.Unlock()
}

{'fraction_non_alphanumeric': 0.13576634275618374, 'fraction_numerical': 0.018551236749116608, 'mean_word_length': 3.2800094517958414, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2660925', 'n_tokens_mistral': 6687, 'n_tokens_neox': 6105, 'n_words': 1685}