<commit_before>//
// command/archive.go
//
// Copyright (c) 2016 <NAME>
//
// This software is released under the MIT License.
//
// http://opensource.org/licenses/mit-license.php
//

package command

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// Archive makes a tar.gz file consists of files maintained a git repository.
func Archive(dir string, filename string) (err error) {

	writeFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return
	}
	defer writeFile.Close()

	writer := bufio.NewWriter(writeFile)
	defer writer.Flush()

	zipWriter, err := gzip.NewWriterLevel(writer, gzip.BestCompression)
	if err != nil {
		return err
	}
	defer zipWriter.Close()

	tarWriter := tar.NewWriter(zipWriter)
	defer tarWriter.Close()

	// Change dir and run.
	cd, err := os.Getwd()
	if err != nil {
		return
	}
	if err = os.Chdir(dir); err != nil {
		return
	}
	defer os.Chdir(cd)

	// Litsing up and write to a tarball.
	ch := make(chan string)
	doneLGS := make(chan error)
	doneTB := make(chan error)

	go listupGitSources(ch, doneLGS)
	go tarballing(tarWriter, ch, doneTB)

	err = <-doneLGS
	close(ch)
	if err != nil {
		return
	}
	return <-doneTB

}

// listupGitSources is a go-routine which lists up git souces and puts
// finding paths to a given ch. After listing up all sources, put nil
// to done. If an error occurs, put the error to done.
func listupGitSources(ch chan<- string, done chan<- error) {

	cmd := exec.Command("git", "ls-files")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		done <- err
		return
	}
	defer stdout.Close()

	doneRL := make(chan error)
	defer close(doneRL)

	go readLine(stdout, ch, doneRL)
	if err = cmd.Run(); err != nil {
		done <- err
		return
	}

	err = <-doneRL
	done <- err
	return

}

// tarballing is a go-routine which write a file given via ch to a tar writer.
// It puts nil to done when it ends. If an error occurs, it puts the error to<commit_msg>Include .git directory to an archive to be sent to a container.<commit_after>//
// command/archive.go
//
// Copyright (c) 2016 <NAME>
//
// This software is released under the MIT License.
//
// http://opensource.org/licenses/mit-license.php
//

package command

import (
	"archive/tar"
	"bufio"
	"compress/gzip"
	"fmt"
	"io"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// Archive makes a tar.gz file consists of files maintained a git repository.
func Archive(dir string, filename string) (err error) {

	writeFile, err := os.OpenFile(filename, os.O_WRONLY|os.O_CREATE, 0600)
	if err != nil {
		return
	}
	defer writeFile.Close()

	writer := bufio.NewWriter(writeFile)
	defer writer.Flush()

	zipWriter, err := gzip.NewWriterLevel(writer, gzip.BestCompression)
	if err != nil {
		return
	}
	defer zipWriter.Close()

	tarWriter := tar.NewWriter(zipWriter)
	defer tarWriter.Close()

	// Change dir and run.
	cd, err := os.Getwd()
	if err != nil {
		return
	}
	if err = os.Chdir(dir); err != nil {
		return
	}
	defer os.Chdir(cd)

	// Litsing up and write to a tarball.
	ch := make(chan string)
	doneLGS := make(chan error)
	doneLGR := make(chan error)
	doneTB := make(chan error)

	go listupGitSources(ch, doneLGS)
	go listupGitRepository(ch, doneLGR)
	go tarballing(tarWriter, ch, doneTB)

	err1 := <-doneLGS
	err2 := <-doneLGR
	close(ch)
	if err1 != nil {
		return err1
	} else if err2 != nil {
		return err2
	}

	return <-doneTB

}

// listupGitSources is a go-routine which lists up git souces and puts
// finding paths to a given ch. After listing up all sources, put nil
// to done. If an error occurs, put the error to done.
func listupGitSources(ch chan<- string, done chan<- error) {

	cmd := exec.Command("git", "ls-files")
	stdout, err := cmd.StdoutPipe()
	if err != nil {
		done <- err
		return
	}
	defer stdout.Close()

	doneRL := make(chan error)
	defer close(doneRL)

	go readLine(stdout, ch, doneRL)
	if err = cmd.Run(); err != nil {
		<-doneRL
		done <- err
		return
	}

	err = <-doneRL
	done <- err
	return

}

// listupGitRepository is a go-routine which lists up git repository
// and puts founded paths to a given ch. After listing up all paths,
// put nil to done. If an error occurs, put the error to done.
func listupGitRepository(ch chan<- string, done chan<- error) {

	err := filepath.Walk(".git", func(path string, info os.FileInfo, err error) error {

		if err != nil {
			return err
		}

		if info.IsDir() {
			return nil
		}

		ch <- path
		return nil

	})

	done <- err

}

// tarballing is a go-routine which write a file given via ch to a tar writer.
// It puts nil to done when it ends. If an error occurs, it puts the error to
{'fraction_non_alphanumeric': 0.12609907784687968, 'fraction_numerical': 0.0047179927085567236, 'mean_word_length': 3.1754700089525514, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '872136', 'n_tokens_mistral': 1770, 'n_tokens_neox': 1647, 'n_words': 565}