<commit_before>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
)

type LocalContainerCreator struct {
	Root string
}

func NewLocalContainerCreator() *LocalContainerCreator {
	r := &LocalContainerCreator{
		Root: filepath.Join(os.TempDir(), "pixelpixel"),
	}
	return r
}

func (lcc *LocalContainerCreator) CreateContainer(fs *tar.Reader, envInjections []string) (Container, error) {
	dir := filepath.Join(lcc.Root, GenerateAlnumString(32))
	err := os.MkdirAll(dir, os.FileMode(0755))
	if err != nil {
		return nil, err
	}

	ctr := &localContainer{
		Root:        dir,
		LogBuffer:   &bytes.Buffer{},
		Terminating: make(chan bool),
	}

	purge := true
	defer func() {
		if purge {
			close(ctr.Terminating)
			ctr.Terminated = true
			os.RemoveAll(dir)
		}
	}()

	err = ctr.extractFileSystem(fs)
	if err != nil {
		return nil, err
	}

	err = ctr.compile()
	if err != nil {
		return ctr, nil
	}

	purge = false
	ctr.Cmd = exec.Command(filepath.Join(dir, "pixel"))
	ctr.Cmd.Dir = dir
	ctr.Cmd.Stdout = ctr.LogBuffer
	ctr.Cmd.Stderr = ctr.LogBuffer
	ctr.Port = <-portGenerator
	ctr.Cmd.Env = stringList(os.Environ(), envInjections, fmt.Sprintf("PORT=%d", ctr.Port))
	err = ctr.Cmd.Start()
	if err != nil {
		return ctr, err
	}

	go func() {
		ctr.Cmd.Wait()
		close(ctr.Terminating)
		ctr.Terminated = true
	}()

	return ctr, nil
}

type localContainer struct {
	Root        string
	Cmd         *exec.Cmd
	Terminating chan bool
	LogBuffer   *bytes.Buffer
	Port        int
	Terminated  bool
}

func (lc *localContainer) IsRunning() bool {
	return !lc.Terminated
}

func (lc *localContainer) Address() net.Addr {
	addr, _ := net.ResolveTCPAddr("tcp4", fmt.Sprintf("localhost:%d", lc.Port))
	return addr
}

func (lc *localContainer) Logs() string {
	return lc.LogBuffer.String()
}

func (lc *localContainer) SoftKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Interrupt)
}

func (lc *localContainer) HardKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Kill)
}

func (lc *localContainer) Wait() {
	select {
	case <-lc.Terminating:
	}
}

func (lc *localContainer) Cleanup() {
	lc.Wait()
	os.RemoveAll(lc.Root)
}

func (lc *localContainer) compile() error {
	files, err := filepath.Glob(filepath.Join(lc.Root, "*.go"))
	if err != nil {
		return err
	}

	cmd := exec.Command("go", stringList("build", "-o", "pixel", files)...)
	cmd.Dir = lc.Root
	cmd.Stdout = lc.LogBuffer
	cmd.Stderr = lc.LogBuffer
	return cmd.Run()
}

func (lc *localContainer) extractFileSystem(fs *tar.Reader) error {
	hdr, err := fs.Next()
	for err != io.EOF {
		if err != nil {
			return err
		}
		file := filepath.Join(lc.Root, hdr.Name)
		switch hdr.Typeflag {
		case tar.TypeDir:
			err := os.MkdirAll(file, os.FileMode(0755))
			if err != nil {
				return err
			}
		case tar.TypeReg:
			err := lc.writeFile(file, fs)
			if err != nil {
				return err
			}
		default:
			log.Printf("Encountered unknown file type 0x%02x, skipping", hdr.Typeflag)
		}

		hdr, err = fs.Next()
	}
	return nil
}<commit_msg>Fetch dependencies before building<commit_after>package main

import (
	"archive/tar"
	"bytes"
	"fmt"
	"io"
	"log"
	"net"
	"os"
	"os/exec"
	"path/filepath"
)

type LocalContainerCreator struct {
	Root string
}

func NewLocalContainerCreator() *LocalContainerCreator {
	r := &LocalContainerCreator{
		Root: filepath.Join(os.TempDir(), "pixelpixel"),
	}
	return r
}

func (lcc *LocalContainerCreator) CreateContainer(fs *tar.Reader, envInjections []string) (Container, error) {
	dir := filepath.Join(lcc.Root, GenerateAlnumString(32))
	err := os.MkdirAll(dir, os.FileMode(0755))
	if err != nil {
		return nil, err
	}

	ctr := &localContainer{
		Root:        dir,
		LogBuffer:   &bytes.Buffer{},
		Terminating: make(chan bool),
	}

	purge := true
	defer func() {
		if purge {
			close(ctr.Terminating)
			ctr.Terminated = true
			os.RemoveAll(dir)
		}
	}()

	err = ctr.extractFileSystem(fs)
	if err != nil {
		return nil, err
	}

	err = ctr.compile()
	if err != nil {
		return ctr, nil
	}

	purge = false
	ctr.Cmd = exec.Command(filepath.Join(dir, "pixel"))
	ctr.Cmd.Dir = dir
	ctr.Cmd.Stdout = ctr.LogBuffer
	ctr.Cmd.Stderr = ctr.LogBuffer
	ctr.Port = <-portGenerator
	ctr.Cmd.Env = stringList(os.Environ(), envInjections, fmt.Sprintf("PORT=%d", ctr.Port))
	err = ctr.Cmd.Start()
	if err != nil {
		return ctr, err
	}

	go func() {
		ctr.Cmd.Wait()
		close(ctr.Terminating)
		ctr.Terminated = true
	}()

	return ctr, nil
}

type localContainer struct {
	Root        string
	Cmd         *exec.Cmd
	Terminating chan bool
	LogBuffer   *bytes.Buffer
	Port        int
	Terminated  bool
}

func (lc *localContainer) IsRunning() bool {
	return !lc.Terminated
}

func (lc *localContainer) Address() net.Addr {
	addr, _ := net.ResolveTCPAddr("tcp4", fmt.Sprintf("localhost:%d", lc.Port))
	return addr
}

func (lc *localContainer) Logs() string {
	return lc.LogBuffer.String()
}

func (lc *localContainer) SoftKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Interrupt)
}

func (lc *localContainer) HardKill() {
	if !lc.IsRunning() || lc.Cmd == nil || lc.Cmd.Process == nil {
		return
	}
	lc.Cmd.Process.Signal(os.Kill)
}

func (lc *localContainer) Wait() {
	select {
	case <-lc.Terminating:
	}
}

func (lc *localContainer) Cleanup() {
	lc.Wait()
	os.RemoveAll(lc.Root)
}

func (lc *localContainer) compile() error {
	cmd := exec.Command("go", "get", "-d")
	cmd.Dir = lc.Root
	cmd.Stdout = lc.LogBuffer
	cmd.Stderr = lc.LogBuffer
	if err := cmd.Run(); err != nil {
		return err
	}

	files, err := filepath.Glob(filepath.Join(lc.Root, "*.go"))
	if err != nil {
		return err
	}

	cmd = exec.Command("go", stringList("build", "-o", "pixel", files)...)
	cmd.Dir = lc.Root
	return cmd.Run()
}

func (lc *localContainer) extractFileSystem(fs *tar.Reader) error {
	hdr, err := fs.Next()
	for err != io.EOF {
		if err != nil {
			return err
		}
		file := filepath.Join(lc.Root, hdr.Name)
		switch hdr.Typeflag {
		case tar.TypeDir:
			err := os.MkdirAll(file, os.FileMode(0755))
			if err != nil {
				return err
			}
		case tar.TypeReg:
			err := lc.writeFile(file, fs)
			if err != nil {
				return err
			}
		default:
			log.Printf("Encountered unknown file type 0x%02x, skipping", hdr.Typeflag)
		}

		hdr, err = fs.Next()
	}
	return nil
}
{'fraction_non_alphanumeric': 0.15592483304861005, 'fraction_numerical': 0.004348501320080758, 'mean_word_length': 3.432209222298692, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2801108', 'n_tokens_mistral': 2628, 'n_tokens_neox': 2498, 'n_words': 574}