<commit_before>package grader

import (
	"compress/gzip"
	"container/list"
	"encoding/json"
	"fmt"
	"github.com/inconshreveable/log15"
	"github.com/lhchavez/quark/common"
	"github.com/lhchavez/quark/runner"
	"io/ioutil"
	"os"
	"path"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"
)

// QueuePriority represents the relative priority of a queue with respect with
// other queues. All the runs in a higher priority queue will be run before
// those in a lower priority queue.
type QueuePriority int

const (
	// QueuePriorityHigh represents a QueuePriority with high priority.
	QueuePriorityHigh = QueuePriority(0)
	// QueuePriorityNormal represents a QueuePriority with normal priority.
	QueuePriorityNormal = QueuePriority(1)
	// QueuePriorityLow represents a QueuePriority with low priority.
	QueuePriorityLow = QueuePriority(2)
	// QueuePriorityEphemeral represents a QueuePriority with the lowest
	// priority. This also does not persist the results in the filesystem.
	QueuePriorityEphemeral = QueuePriority(3)

	// QueueCount represents the total number of queues in the grader.
	QueueCount = 4

	// DefaultQueueName is the default queue name.
	DefaultQueueName = "default"
)

// A EphemeralRunRequest represents a client's request to run some code.
type EphemeralRunRequest struct {
	Source   string               `json:"source"`
	Language string               `json:"language"`
	Input    *common.LiteralInput `json:"input"`
}

// EphemeralRunManager handles a queue of recently-submitted ephemeral runs.
// This has a fixed maximum size with a last-in, first-out eviction policy.
type EphemeralRunManager struct {
	sync.Mutex
	evictList *list.List
	ctx       *Context
	paths     map[string]struct{}
	totalSize int64
	sizeLimit int64
}

type ephemeralRunEntry struct {
	path string
	size int64
}

func directorySize(root string) (result int64, err error) {
	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			// Bail out the whole operation upon the first error.
			return err
		}
		result += info.Size()
		return nil
	})

	return
}

// NewEphemeralRunManager returns a new EphemeralRunManager.
func NewEphemeralRunManager(ctx *Context) *EphemeralRunManager {
	return &EphemeralRunManager{
		evictList: list.New(),
		ctx:       ctx,
		paths:     make(map[string]struct{}),
		sizeLimit: ctx.Config.Grader.Ephemeral.EphemeralSizeLimit,
	}
}

// Initialize goes through the past ephemeral runs in order to add them to the
// FIFO cache.
func (mgr *EphemeralRunManager) Initialize() error {
	ephemeralPath := path.Join(mgr.ctx.Config.Grader.RuntimePath, "ephemeral")

	if err := os.MkdirAll(ephemeralPath, 0755); err != nil {
		return err
	}

	dir, err := os.Open(ephemeralPath)
	if err != nil {
		return err
	}

	entryNames, err := dir.Readdirnames(0)
	if err != nil {
		return err
	}

	var firstErr error
	for _, entryName := range entryNames {
		entryPath := path.Join(ephemeralPath, entryName)
		size, err := directorySize(entryPath)
		if err != nil {
			if firstErr == nil {
				firstErr = err
			}
			// This errored out. Let's try to remove the directory since it's not
			// going to participate in the total size.
			if err = os.RemoveAll(entryPath); err != nil {
				mgr.ctx.Log.Error(
					"Failed to remove directory during initialization",
					"entry", entryPath,
					"err", err,
				)
			}
			continue
		}

		if err = mgr.add(&ephemeralRunEntry{
			path: entryPath,
			size: size,
		}); err != nil {
			mgr.ctx.Log.Error(
				"Failed to add entry to manager",
				"entry", entryPath,
				"err", err,
			)
			if firstErr == nil {
				firstErr = err
			}
		}
	}

	return firstErr
}

// SetEphemeral makes a RunContext ephemeral. It does this by setting its
// runtime path to be in the ephemeral subdirectory.
func (mgr *EphemeralRunManager) SetEphemeral(runCtx *RunContext) (string, error) {
	ephemeralPath := path.Join(mgr.ctx.Config.Grader.RuntimePath, "ephemeral")

	var err error
	if runCtx.GradeDir, err = ioutil.TempDir(ephemeralPath, ""); err != nil {
		return "", err
	}
	return path.Base(runCtx.GradeDir), nil
}

// Commit adds the files produced by the RunContext into the FIFO cache,
// potentially evicting older runs in the process.
func (mgr *EphemeralRunManager) Commit(runCtx *RunContext) error {
	size, err := directorySize(runCtx.GradeDir)
	if err != nil {
		return err
	}

	return mgr.add(&ephemeralRunEntry{
		path: runCtx.GradeDir,
		size: size,
	})
}

// Get returns the filesystem path for a previous ephemeral run, and whether or
// not it actually exists.
func (mgr *EphemeralRunManager) Get(token string) (string, bool) {
	ephemeralPath := path.Join(mgr.ctx.Config.Grader.RuntimePath, "ephemeral")

	mgr.Lock()
	defer mgr.Unlock()

	entryPath := path.Join(ephemeralPath, token)<commit_msg>Make the names of the ephemeral runs less predictable<commit_after>package grader

import (
	"compress/gzip"
	"container/list"
	"encoding/json"
	"fmt"
	"github.com/inconshreveable/log15"
	"github.com/lhchavez/quark/common"
	"github.com/lhchavez/quark/runner"
	"io/ioutil"
	"math/rand"
	"os"
	"path"
	"path/filepath"
	"sync"
	"sync/atomic"
	"time"
)

// QueuePriority represents the relative priority of a queue with respect with
// other queues. All the runs in a higher priority queue will be run before
// those in a lower priority queue.
type QueuePriority int

const (
	// QueuePriorityHigh represents a QueuePriority with high priority.
	QueuePriorityHigh = QueuePriority(0)
	// QueuePriorityNormal represents a QueuePriority with normal priority.
	QueuePriorityNormal = QueuePriority(1)
	// QueuePriorityLow represents a QueuePriority with low priority.
	QueuePriorityLow = QueuePriority(2)
	// QueuePriorityEphemeral represents a QueuePriority with the lowest
	// priority. This also does not persist the results in the filesystem.
	QueuePriorityEphemeral = QueuePriority(3)

	// QueueCount represents the total number of queues in the grader.
	QueueCount = 4

	// DefaultQueueName is the default queue name.
	DefaultQueueName = "default"
)

// A EphemeralRunRequest represents a client's request to run some code.
type EphemeralRunRequest struct {
	Source   string               `json:"source"`
	Language string               `json:"language"`
	Input    *common.LiteralInput `json:"input"`
}

// EphemeralRunManager handles a queue of recently-submitted ephemeral runs.
// This has a fixed maximum size with a last-in, first-out eviction policy.
type EphemeralRunManager struct {
	sync.Mutex
	evictList *list.List
	ctx       *Context
	rand      *rand.Rand
	paths     map[string]struct{}
	totalSize int64
	sizeLimit int64
}

type ephemeralRunEntry struct {
	path string
	size int64
}

func directorySize(root string) (result int64, err error) {
	err = filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			// Bail out the whole operation upon the first error.
			return err
		}
		result += info.Size()
		return nil
	})

	return
}

// NewEphemeralRunManager returns a new EphemeralRunManager.
func NewEphemeralRunManager(ctx *Context) *EphemeralRunManager {
	return &EphemeralRunManager{
		evictList: list.New(),
		ctx:       ctx,
		rand:      rand.New(rand.NewSource(time.Now().UnixNano())),
		paths:     make(map[string]struct{}),
		sizeLimit: ctx.Config.Grader.Ephemeral.EphemeralSizeLimit,
	}
}

// Initialize goes through the past ephemeral runs in order to add them to the
// FIFO cache.
func (mgr *EphemeralRunManager) Initialize() error {
	ephemeralPath := path.Join(mgr.ctx.Config.Grader.RuntimePath, "ephemeral")

	if err := os.MkdirAll(ephemeralPath, 0755); err != nil {
		return err
	}

	dir, err := os.Open(ephemeralPath)
	if err != nil {
		return err
	}

	entryNames, err := dir.Readdirnames(0)
	if err != nil {
		return err
	}

	var firstErr error
	for _, entryName := range entryNames {
		entryPath := path.Join(ephemeralPath, entryName)
		size, err := directorySize(entryPath)
		if err != nil {
			if firstErr == nil {
				firstErr = err
			}
			// This errored out. Let's try to remove the directory since it's not
			// going to participate in the total size.
			if err = os.RemoveAll(entryPath); err != nil {
				mgr.ctx.Log.Error(
					"Failed to remove directory during initialization",
					"entry", entryPath,
					"err", err,
				)
			}
			continue
		}

		if err = mgr.add(&ephemeralRunEntry{
			path: entryPath,
			size: size,
		}); err != nil {
			mgr.ctx.Log.Error(
				"Failed to add entry to manager",
				"entry", entryPath,
				"err", err,
			)
			if firstErr == nil {
				firstErr = err
			}
		}
	}

	return firstErr
}

func (mgr *EphemeralRunManager) tempDir() (name string, err error) {
	ephemeralPath := path.Join(mgr.ctx.Config.Grader.RuntimePath, "ephemeral")
	buf := make([]byte, 10)

	for i := 0; i < 10000; i++ {
		mgr.rand.Read(buf)
		try := path.Join(ephemeralPath, fmt.Sprintf("%x", buf))
		err = os.Mkdir(try, 0700)
		if os.IsExist(err) {
			continue
		}
		if os.IsNotExist(err) {
			if _, err := os.Stat(ephemeralPath); os.IsNotExist(err) {
				return "", err
			}
		}
		if err == nil {
			name = try
		}
		break
	}

	return
}

// SetEphemeral makes a RunContext ephemeral. It does this by setting its
// runtime path to be in the ephemeral subdirectory.
func (mgr *EphemeralRunManager) SetEphemeral(runCtx *RunContext) (string, error) {
	var err error
	if runCtx.GradeDir, err = mgr.tempDir(); err != nil {
		return "", err
	}
	return path.Base(runCtx.GradeDir), nil
}

// Commit adds the files produced by the RunContext into the FIFO cache,
// potentially evicting older runs in the process.
func (mgr *EphemeralRunManager) Commit(runCtx *RunContext) error {
	size, err := directorySize(runCtx.GradeDir)
	if err != nil {
		return err
	}

	return mgr.add(&ephemeralRunEntry{
		path: runCtx.GradeDir,
		size: size,
	})
}

// Get returns the filesystem path for a previous ephemeral run, and whether or
// not it actually exists.
func (mgr *EphemeralRunManager) Get(token string) (string, bool) {
	ephemeralPath := path.Join(mgr.ctx.Config.Grader.RuntimePath, "ephemeral")

	mgr.Lock()
	defer mgr.Unlock()

	entryPath := path.Join(ephemeralPath, token)
{'fraction_non_alphanumeric': 0.10423452768729642, 'fraction_numerical': 0.005132760833086566, 'mean_word_length': 3.6455754241173772, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7270904', 'n_tokens_mistral': 3707, 'n_tokens_neox': 3375, 'n_words': 1021}