<commit_before>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

func templId(templ, arch, version string, kurjun *http.Client) string {
	url := config.Management.Kurjun + "/template/info?name=" + templ + "&version=" + version + "&type=text"
	if version == "stable" || len(version) == 0 {
		url = config.Management.Kurjun + "/template/info?name=" + templ + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug(config.Management.Kurjun + "/template/info?name=" + templ + "&type=text")
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(templ, md5, arch string) string {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		file := strings.Split(f.Name(), "-subutai-template_")
		if len(file) == 2 && file[0] == templ && strings.Contains(file[1], arch) {
			if len(md5) == 0 {
				fmt.Print("Cannot check md5 of local template. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
				}
			}
			if md5 == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
			}
		}
	}
	return ""
}

func download(file, id string, kurjun *http.Client) string {
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + file)
	log.Check(log.FatalLevel, "Creating file "+file, err)
	defer out.Close()
	response, err := kurjun.Get(config.Management.Kurjun + "/template/get?id=" + id)
	log.Check(log.FatalLevel, "Getting "+config.Management.Kurjun+"/template/get?id="+id, err)
	defer response.Body.Close()
	_, err = io.Copy(out, response.Body)
	log.Check(log.FatalLevel, "Writing file "+file, err)
	if strings.Split(id, ".")[1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+file) {
		return config.Agent.LxcPrefix + "tmpdir/" + file
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return ""
}
func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(templ, version, token string) {
	log.Info("Importing " + templ)
	for !lockSubutai(templ + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()<commit_msg>Added progress bar for import. #388<commit_after>package lib

import (
	"crypto/md5"
	"fmt"
	"io"
	"io/ioutil"
	"net/http"
	"os"
	"runtime"
	"strings"
	"time"

	"github.com/cheggaaa/pb"
	"github.com/nightlyone/lockfile"
	"github.com/pivotal-golang/archiver/extractor"

	"github.com/subutai-io/base/agent/config"
	"github.com/subutai-io/base/agent/lib/container"
	"github.com/subutai-io/base/agent/lib/gpg"
	"github.com/subutai-io/base/agent/lib/template"
	"github.com/subutai-io/base/agent/log"
)

var (
	lock lockfile.Lockfile
)

func templId(templ, arch, version string, kurjun *http.Client) string {
	url := config.Management.Kurjun + "/template/info?name=" + templ + "&version=" + version + "&type=text"
	if version == "stable" || len(version) == 0 {
		url = config.Management.Kurjun + "/template/info?name=" + templ + "&type=text"
	}
	response, err := kurjun.Get(url)
	log.Debug(config.Management.Kurjun + "/template/info?name=" + templ + "&type=text")
	if log.Check(log.WarnLevel, "Getting kurjun response", err) || response.StatusCode != 200 {
		return ""
	}

	hash, err := ioutil.ReadAll(response.Body)
	response.Body.Close()
	if log.Check(log.WarnLevel, "Reading response body", err) {
		return ""
	}

	log.Debug("Template id: " + string(hash))
	return string(hash)
}

func md5sum(filePath string) string {
	file, err := os.Open(filePath)
	if err != nil {
		return ""
	}
	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ""
	}
	return fmt.Sprintf("%x", hash.Sum(nil))
}

func checkLocal(templ, md5, arch string) string {
	var response string
	files, _ := ioutil.ReadDir(config.Agent.LxcPrefix + "tmpdir")
	for _, f := range files {
		file := strings.Split(f.Name(), "-subutai-template_")
		if len(file) == 2 && file[0] == templ && strings.Contains(file[1], arch) {
			if len(md5) == 0 {
				fmt.Print("Cannot check md5 of local template. Trust anyway? (y/n)")
				_, err := fmt.Scanln(&response)
				log.Check(log.FatalLevel, "Reading input", err)
				if response == "y" {
					return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
				}
			}
			if md5 == md5sum(config.Agent.LxcPrefix+"tmpdir/"+f.Name()) {
				return config.Agent.LxcPrefix + "tmpdir/" + f.Name()
			}
		}
	}
	return ""
}

func download(file, id string, kurjun *http.Client) string {
	out, err := os.Create(config.Agent.LxcPrefix + "tmpdir/" + file)
	log.Check(log.FatalLevel, "Creating file "+file, err)
	defer out.Close()
	response, err := kurjun.Get(config.Management.Kurjun + "/template/get?id=" + id)
	log.Check(log.FatalLevel, "Getting "+config.Management.Kurjun+"/template/get?id="+id, err)
	defer response.Body.Close()

	bar := pb.New(int(response.ContentLength)).SetUnits(pb.U_BYTES)
	bar.Start()
	rd := bar.NewProxyReader(response.Body)

	_, err = io.Copy(out, rd)
	log.Check(log.FatalLevel, "Writing file "+file, err)
	if strings.Split(id, ".")[1] == md5sum(config.Agent.LxcPrefix+"tmpdir/"+file) {
		return config.Agent.LxcPrefix + "tmpdir/" + file
	}
	log.Error("Failed to check MD5 after download. Please check your connection and try again.")
	return ""
}
func lockSubutai(file string) bool {
	lock, err := lockfile.New("/var/run/lock/subutai." + file)
	if log.Check(log.DebugLevel, "Init lock "+file, err) {
		return false
	}

	err = lock.TryLock()
	if log.Check(log.DebugLevel, "Locking file "+file, err) {
		return false
	}

	return true
}

func unlockSubutai() {
	lock.Unlock()
}

func LxcImport(templ, version, token string) {
	log.Info("Importing " + templ)
	for !lockSubutai(templ + ".import") {
		time.Sleep(time.Second * 1)
	}
	defer unlockSubutai()
{'fraction_non_alphanumeric': 0.1747227750924083, 'fraction_numerical': 0.006113164628945123, 'mean_word_length': 4.032188841201717, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3349763', 'n_tokens_mistral': 2804, 'n_tokens_neox': 2659, 'n_words': 546}