<commit_before>package main

import (
	"fmt"
	"io"
	"io/ioutil"
	"log"
	"os"
	"strconv"
	"strings"

	docker "github.com/bywan/go-dockercommand"
)

type Builder struct {
	Options *BuildOptions
}

type BuildOptions struct {
	DockerfileFolder string
	SourceFolder     string
	ProjectID        string
	JobID            string
}

type BuiltImage struct {
	Image     string
	VariantID int
}

func (b *Builder) Build() ([]BuiltImage, error) {

	log.Printf("Starting building Dockerfiles\n")
	files, err := listBuildfiles(b.Options.DockerfileFolder)
	if err != nil {
		return nil, err
	}

	client, err := docker.NewDocker(DockerEndpoint)
	if err != nil {
		return nil, err
	}

	errChan := make(chan error)
	successChan := make(chan BuiltImage)
	remainingBuilds := len(files)

	for i, file := range files {
		go buildContainer(client, i, b, file, successChan, errChan)
	}

	var buildImages []BuiltImage
	for {
		select {
		case tag := <-successChan:
			buildImages = append(buildImages, tag)
			remainingBuilds--
		case err := <-errChan:
			return nil, err
		}

		if remainingBuilds == 0 {
			break
		}
	}
	return buildImages, nil
}

func buildContainer(client *docker.Docker, variantID int, b *Builder, file *buildFiles, successChan chan BuiltImage, errChan chan error) {
	for _, buildFile := range file.BuildFiles {
		splitString := strings.Split(buildFile, "/")
		CopyFile(buildFile, fmt.Sprintf("%s/%s", b.Options.SourceFolder, splitString[len(splitString)-1]))
	}

	tag := fmt.Sprintf("bazooka/build-%s-%s-%d", b.Options.ProjectID, b.Options.JobID, variantID)
	err := client.Build(&docker.BuildOptions{
		Tag:        tag,
		Dockerfile: file.Dockerfile,
		Path:       b.Options.SourceFolder,
	})
	if err != nil {
		errChan <- err
	} else {
		successChan <- BuiltImage{
			Image:     tag,
			VariantID: variantID,
		}
	}
}

func listBuildfiles(source string) ([]*buildFiles, error) {
	files, err := ioutil.ReadDir(source)
	if err != nil {
		return nil, err
	}
	var output []*buildFiles
	for _, file := range files {
		if file.Mode().IsDir() {
			index, err := strconv.ParseInt(file.Name(), 10, 64)
			if err != nil {
				return nil, err
			}
			filesBuild, err := ioutil.ReadDir(fmt.Sprintf("%s/%s", source, file.Name()))
			if err != nil {
				return nil, err
			}
			var result []string
			for _, fileBuild := range filesBuild {
				if fileBuild.Name() != "Dockerfile" {
					result = append(result, fmt.Sprintf("%s/%s/%s", source, file.Name(), fileBuild.Name()))
				}
			}
			output = append(output, &buildFiles{
				Dockerfile: fmt.Sprintf("%s/%s/Dockerfile", source, file.Name()),
				BuildFiles: result,
				JobIndex:   index,
			})

		}
	}
	return output, nil
}

type buildFiles struct {
	Dockerfile string
	BuildFiles []string
	JobIndex   int64
}

// TODO Extract this or replace it
func CopyFile(src, dst string) (err error) {
	sfi, err := os.Stat(src)
	if err != nil {
		return
	}
	if !sfi.Mode().IsRegular() {
		// cannot copy non-regular files (e.g., directories,
		// symlinks, devices, etc.)
		return fmt.Errorf("CopyFile: non-regular source file %s (%q)", sfi.Name(), sfi.Mode().String())
	}
	dfi, err := os.Stat(dst)
	if err != nil {
		if !os.IsNotExist(err) {
			return
		}
	} else {
		if !(dfi.Mode().IsRegular()) {
			return fmt.Errorf("CopyFile: non-regular destination file %s (%q)", dfi.Name(), dfi.Mode().String())
		}
		if os.SameFile(sfi, dfi) {
			return
		}
	}
	if err = os.Link(src, dst); err == nil {
		return
	}
	err = copyFileContents(src, dst)
	return
}

// copyFileContents copies the contents of the file named src to the file named
// by dst. The file will be created if it does not already exist. If the
// destination file exists, all it's contents will be replaced by the contents
// of the source file.
func copyFileContents(src, dst string) (err error) {
	in, err := os.Open(src)
	if err != nil {
		return
	}
	defer in.Close()
	out, err := os.Create(dst)
	if err != nil {
		return
	}
	defer func() {
		cerr := out.Close()
		if err == nil {
			err = cerr
		}
	}()
	if _, err = io.Copy(out, in); err != nil {
		return
	}
	err = out.Sync()
	return
}<commit_msg>use commons method<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"
	"strconv"
	"strings"

	docker "github.com/bywan/go-dockercommand"
	lib "github.com/haklop/bazooka/commons"
)

type Builder struct {
	Options *BuildOptions
}

type BuildOptions struct {
	DockerfileFolder string
	SourceFolder     string
	ProjectID        string
	JobID            string
}

type BuiltImage struct {
	Image     string
	VariantID int
}

func (b *Builder) Build() ([]BuiltImage, error) {

	log.Printf("Starting building Dockerfiles\n")
	files, err := listBuildfiles(b.Options.DockerfileFolder)
	if err != nil {
		return nil, err
	}

	client, err := docker.NewDocker(DockerEndpoint)
	if err != nil {
		return nil, err
	}

	errChan := make(chan error)
	successChan := make(chan BuiltImage)
	remainingBuilds := len(files)

	for i, file := range files {
		go buildContainer(client, i, b, file, successChan, errChan)
	}

	var buildImages []BuiltImage
	for {
		select {
		case tag := <-successChan:
			buildImages = append(buildImages, tag)
			remainingBuilds--
		case err := <-errChan:
			return nil, err
		}

		if remainingBuilds == 0 {
			break
		}
	}
	return buildImages, nil
}

func buildContainer(client *docker.Docker, variantID int, b *Builder, file *buildFiles, successChan chan BuiltImage, errChan chan error) {
	for _, buildFile := range file.BuildFiles {
		splitString := strings.Split(buildFile, "/")
		lib.CopyFile(buildFile, fmt.Sprintf("%s/%s", b.Options.SourceFolder, splitString[len(splitString)-1]))
	}

	tag := fmt.Sprintf("bazooka/build-%s-%s-%d", b.Options.ProjectID, b.Options.JobID, variantID)
	err := client.Build(&docker.BuildOptions{
		Tag:        tag,
		Dockerfile: file.Dockerfile,
		Path:       b.Options.SourceFolder,
	})
	if err != nil {
		errChan <- err
	} else {
		successChan <- BuiltImage{
			Image:     tag,
			VariantID: variantID,
		}
	}
}

func listBuildfiles(source string) ([]*buildFiles, error) {
	files, err := ioutil.ReadDir(source)
	if err != nil {
		return nil, err
	}
	var output []*buildFiles
	for _, file := range files {
		if file.Mode().IsDir() {
			index, err := strconv.ParseInt(file.Name(), 10, 64)
			if err != nil {
				return nil, err
			}
			filesBuild, err := ioutil.ReadDir(fmt.Sprintf("%s/%s", source, file.Name()))
			if err != nil {
				return nil, err
			}
			var result []string
			for _, fileBuild := range filesBuild {
				if fileBuild.Name() != "Dockerfile" {
					result = append(result, fmt.Sprintf("%s/%s/%s", source, file.Name(), fileBuild.Name()))
				}
			}
			output = append(output, &buildFiles{
				Dockerfile: fmt.Sprintf("%s/%s/Dockerfile", source, file.Name()),
				BuildFiles: result,
				JobIndex:   index,
			})

		}
	}
	return output, nil
}

type buildFiles struct {
	Dockerfile string
	BuildFiles []string
	JobIndex   int64
}
{'fraction_non_alphanumeric': 0.14016250725478815, 'fraction_numerical': 0.0023215322112594312, 'mean_word_length': 3.357142857142857, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4858831', 'n_tokens_mistral': 2660, 'n_tokens_neox': 2392, 'n_words': 645}