<commit_before>package main

//go:generate go-bindata -nomemcopy data/dist/...

import (
	"bytes"
	"compress/gzip"
	"database/sql"
	"encoding/base64"
	"expvar"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	_ "github.com/mattn/go-sqlite3"
	"github.com/omegaup/quark/common"
	"github.com/omegaup/quark/grader"
	"io"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"path/filepath"
	"sync"
	"sync/atomic"
)

var (
	version    = flag.Bool("version", false, "Print the version and exit")
	insecure   = flag.Bool("insecure", false, "Do not use TLS")
	skipAssets = flag.Bool("skip-assets", false, "Do not use pre-packaged assets")
	configPath = flag.String(
		"config",
		"/etc/omegaup/grader/config.json",
		"Grader configuration file",
	)
	globalContext atomic.Value
	server        *http.Server

	// ProgramVersion is the version of the code from which the binary was built from.
	ProgramVersion string
)

type processRunStatus struct {
	status int
	retry  bool
}

// A ResponseStruct represents the result of a run request.
type ResponseStruct struct {
	Results  string
	Logs     string
	FilesZip string
	Tracing  string
}

func loadContext() error {
	f, err := os.Open(*configPath)
	if err != nil {
		return err
	}
	defer f.Close()

	ctx, err := grader.NewContext(f)
	if err != nil {
		return err
	}
	globalContext.Store(ctx)
	return nil
}

func context() *grader.Context {
	return globalContext.Load().(*grader.Context)
}

func peerName(r *http.Request) string {
	if *insecure {
		return r.RemoteAddr
	}
	return r.TLS.PeerCertificates[0].Subject.CommonName
}

func readGzippedFile(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	gz, err := gzip.NewReader(f)
	if err != nil {
		return "", err
	}
	defer gz.Close()

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, gz); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func readBase64File(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	var buf bytes.Buffer
	enc := base64.NewEncoder(base64.StdEncoding, &buf)
	if err != nil {
		return "", err
	}

	if _, err := io.Copy(enc, f); err != nil {
		return "", err
	}
	enc.Close()
	return buf.String(), nil
}

type wrappedFileSystem struct {
	fileSystem http.FileSystem
}

func (fs *wrappedFileSystem) Open(name string) (http.File, error) {
	if *skipAssets {
		path := "/data" + filepath.Clean(filepath.Join("/", name))
		return os.Open(path)
	}
	if file, err := fs.fileSystem.Open(name); err == nil {
		return file, nil
	}
	return nil, os.ErrNotExist
}

func queEventsProcessor(events <-chan *grader.QueueEvent) {
	ctx := context()
	for {
		select {
		case event, ok := <-events:
			if !ok {
				return
			}

			switch event.Type {
			case grader.QueueEventTypeManagerAdded:
				ctx.Metrics.GaugeAdd("grader_queue_total_length", 1)
			case grader.QueueEventTypeManagerRemoved:
				ctx.Metrics.GaugeAdd("grader_queue_total_length", -1)
				ctx.Metrics.SummaryObserve("grader_queue_delay_seconds", event.Delta.Seconds())
			case grader.QueueEventTypeQueueRemoved:
				switch event.Priority {
				case grader.QueuePriorityEphemeral:
					ctx.Metrics.SummaryObserve("grader_queue_ephemeral_delay_seconds", event.Delta.Seconds())
				case grader.QueuePriorityLow:
					ctx.Metrics.SummaryObserve("grader_queue_low_delay_seconds", event.Delta.Seconds())
				case grader.QueuePriorityNormal:
					ctx.Metrics.SummaryObserve("grader_queue_normal_delay_seconds", event.Delta.Seconds())
				case grader.QueuePriorityHigh:
					ctx.Metrics.SummaryObserve("grader_queue_high_delay_seconds", event.Delta.Seconds())
				}
			case grader.QueueEventTypeRetried:
				ctx.Metrics.GaugeAdd("grader_runs_retry", 1)
			case grader.QueueEventTypeAbandoned:
				ctx.Metrics.GaugeAdd("grader_runs_abandoned", 1)
			}
		}
	}
}

func main() {
	flag.Parse()

	if *version {
		fmt.Printf("omegaup-grader %s\n", ProgramVersion)<commit_msg>Cleanly shut down the git library upon process exit<commit_after>package main

//go:generate go-bindata -nomemcopy data/dist/...

import (
	"bytes"
	"compress/gzip"
	"database/sql"
	"encoding/base64"
	"expvar"
	"flag"
	"fmt"
	_ "github.com/go-sql-driver/mysql"
	git "github.com/lhchavez/git2go"
	_ "github.com/mattn/go-sqlite3"
	"github.com/omegaup/quark/common"
	"github.com/omegaup/quark/grader"
	"io"
	"net/http"
	_ "net/http/pprof"
	"os"
	"path"
	"path/filepath"
	"sync"
	"sync/atomic"
)

var (
	version    = flag.Bool("version", false, "Print the version and exit")
	insecure   = flag.Bool("insecure", false, "Do not use TLS")
	skipAssets = flag.Bool("skip-assets", false, "Do not use pre-packaged assets")
	configPath = flag.String(
		"config",
		"/etc/omegaup/grader/config.json",
		"Grader configuration file",
	)
	globalContext atomic.Value
	server        *http.Server

	// ProgramVersion is the version of the code from which the binary was built from.
	ProgramVersion string
)

type processRunStatus struct {
	status int
	retry  bool
}

// A ResponseStruct represents the result of a run request.
type ResponseStruct struct {
	Results  string
	Logs     string
	FilesZip string
	Tracing  string
}

func loadContext() error {
	f, err := os.Open(*configPath)
	if err != nil {
		return err
	}
	defer f.Close()

	ctx, err := grader.NewContext(f)
	if err != nil {
		return err
	}
	globalContext.Store(ctx)
	return nil
}

func context() *grader.Context {
	return globalContext.Load().(*grader.Context)
}

func peerName(r *http.Request) string {
	if *insecure {
		return r.RemoteAddr
	}
	return r.TLS.PeerCertificates[0].Subject.CommonName
}

func readGzippedFile(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	gz, err := gzip.NewReader(f)
	if err != nil {
		return "", err
	}
	defer gz.Close()

	var buf bytes.Buffer
	if _, err := io.Copy(&buf, gz); err != nil {
		return "", err
	}
	return buf.String(), nil
}

func readBase64File(filename string) (string, error) {
	f, err := os.Open(filename)
	if err != nil {
		return "", err
	}
	defer f.Close()

	var buf bytes.Buffer
	enc := base64.NewEncoder(base64.StdEncoding, &buf)
	if err != nil {
		return "", err
	}

	if _, err := io.Copy(enc, f); err != nil {
		return "", err
	}
	enc.Close()
	return buf.String(), nil
}

type wrappedFileSystem struct {
	fileSystem http.FileSystem
}

func (fs *wrappedFileSystem) Open(name string) (http.File, error) {
	if *skipAssets {
		path := "/data" + filepath.Clean(filepath.Join("/", name))
		return os.Open(path)
	}
	if file, err := fs.fileSystem.Open(name); err == nil {
		return file, nil
	}
	return nil, os.ErrNotExist
}

func queEventsProcessor(events <-chan *grader.QueueEvent) {
	ctx := context()
	for {
		select {
		case event, ok := <-events:
			if !ok {
				return
			}

			switch event.Type {
			case grader.QueueEventTypeManagerAdded:
				ctx.Metrics.GaugeAdd("grader_queue_total_length", 1)
			case grader.QueueEventTypeManagerRemoved:
				ctx.Metrics.GaugeAdd("grader_queue_total_length", -1)
				ctx.Metrics.SummaryObserve("grader_queue_delay_seconds", event.Delta.Seconds())
			case grader.QueueEventTypeQueueRemoved:
				switch event.Priority {
				case grader.QueuePriorityEphemeral:
					ctx.Metrics.SummaryObserve("grader_queue_ephemeral_delay_seconds", event.Delta.Seconds())
				case grader.QueuePriorityLow:
					ctx.Metrics.SummaryObserve("grader_queue_low_delay_seconds", event.Delta.Seconds())
				case grader.QueuePriorityNormal:
					ctx.Metrics.SummaryObserve("grader_queue_normal_delay_seconds", event.Delta.Seconds())
				case grader.QueuePriorityHigh:
					ctx.Metrics.SummaryObserve("grader_queue_high_delay_seconds", event.Delta.Seconds())
				}
			case grader.QueueEventTypeRetried:
				ctx.Metrics.GaugeAdd("grader_runs_retry", 1)
			case grader.QueueEventTypeAbandoned:
				ctx.Metrics.GaugeAdd("grader_runs_abandoned", 1)
			}
		}
	}
}

func main() {
	defer git.Shutdown()

	flag.Parse()

	if *version {
		fmt.Printf("omegaup-grader %s\n", ProgramVersion)
{'fraction_non_alphanumeric': 0.13215309811744172, 'fraction_numerical': 0.0036155092881186885, 'mean_word_length': 3.909424724602203, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5493870', 'n_tokens_mistral': 3099, 'n_tokens_neox': 2859, 'n_words': 641}