<commit_before>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"
	"gopkg.in/yaml.v2"
)

const localDirKey = "localDir"
const flagRequired = "REQUIRED"

var once sync.Once
var cacheCredentialsPtr *bool

var rand uint32
var randmu sync.Mutex

func reseed() uint32 {
	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
}

func nextSuffix() string {
	randmu.Lock()
	r := rand
	if r == 0 {
		r = reseed()
	}
	r = r*1664525 + 1013904223 // constants from Numerical Recipes
	rand = r
	randmu.Unlock()
	return strconv.Itoa(int(1e9 + r%1e9))[1:]
}

func p(text string, cr color.Attribute) {
	color.New(color.FgBlack, color.Bold, color.BgWhite).Printf("=>")
	fmt.Printf(" ")
	color.New(cr, color.Bold).Println(text)
}

func run(args ...string) (string, error) {
	getStdout := args[0] == "_return_stdout_"
	if getStdout {
		args = args[1:]
	}
	p("About to run command", color.FgGreen)
	p("$ "+strings.Join(args, " "), color.FgRed)
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stderr = os.Stderr
	if getStdout {
		bs, err := cmd.Output()
		return string(bs), err
	} else {
		cmd.Stdout = os.Stdout
		err := cmd.Run()
		return "", err
	}
}

func build(name string, path string) {
	path = expandTilde(path)
	finfo, _ := os.Stat(path)
	if finfo == nil {
		color.Red("dir ", path, " does not exist => not building", name, "\n")
		return
	}
	if !finfo.IsDir() {
		color.Red("file ", path, " is not a directory => not building", name, "\n")
		return
	}
	run("docker", "build", "-t", name, path)

}

func expandTilde(ppath string) string {
	if ppath[:2] == "~/" {
		usr, _ := user.Current()
		dir := usr.HomeDir
		return strings.Replace(ppath, "~", dir, 1)
	}
	return ppath
}

func likeAlmostTheSame(url1 string, url2 string) bool {
	purl1, _ := url.Parse(url1)
	purl2, _ := url.Parse(url2)
	if purl1.Host == purl2.Host && purl1.Path == purl2.Path {
		return true
	}
	return false
}

func get(repo map[string]string, repoUrl string) error {
	p("About to git clone/pull repo:", color.FgGreen)<commit_msg>:scream: be aware of .git suffix in URL paths of repositories<commit_after>package main

import (
	"errors"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"net/url"
	"os"
	"os/exec"
	"os/user"
	"path"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/fatih/color"
	"gopkg.in/yaml.v2"
)

const localDirKey = "localDir"
const flagRequired = "REQUIRED"

var once sync.Once
var cacheCredentialsPtr *bool

var rand uint32
var randmu sync.Mutex

func reseed() uint32 {
	return uint32(time.Now().UnixNano() + int64(os.Getpid()))
}

func nextSuffix() string {
	randmu.Lock()
	r := rand
	if r == 0 {
		r = reseed()
	}
	r = r*1664525 + 1013904223 // constants from Numerical Recipes
	rand = r
	randmu.Unlock()
	return strconv.Itoa(int(1e9 + r%1e9))[1:]
}

func p(text string, cr color.Attribute) {
	color.New(color.FgBlack, color.Bold, color.BgWhite).Printf("=>")
	fmt.Printf(" ")
	color.New(cr, color.Bold).Println(text)
}

func run(args ...string) (string, error) {
	getStdout := args[0] == "_return_stdout_"
	if getStdout {
		args = args[1:]
	}
	p("About to run command", color.FgGreen)
	p("$ "+strings.Join(args, " "), color.FgRed)
	cmd := exec.Command(args[0], args[1:]...)
	cmd.Stderr = os.Stderr
	if getStdout {
		bs, err := cmd.Output()
		return string(bs), err
	} else {
		cmd.Stdout = os.Stdout
		err := cmd.Run()
		return "", err
	}
}

func build(name string, path string) {
	path = expandTilde(path)
	finfo, _ := os.Stat(path)
	if finfo == nil {
		color.Red("dir ", path, " does not exist => not building", name, "\n")
		return
	}
	if !finfo.IsDir() {
		color.Red("file ", path, " is not a directory => not building", name, "\n")
		return
	}
	run("docker", "build", "-t", name, path)

}

func expandTilde(ppath string) string {
	if ppath[:2] == "~/" {
		usr, _ := user.Current()
		dir := usr.HomeDir
		return strings.Replace(ppath, "~", dir, 1)
	}
	return ppath
}

func likeAlmostTheSame(url1 string, url2 string) bool {
	purl1, _ := url.Parse(url1)
	purl2, _ := url.Parse(url2)
	if purl1.Host == purl2.Host {
		if strings.TrimSuffix(purl1.Path, ".git") == strings.TrimSuffix(purl2.Path, ".git") {
			return true
		}
	}
	return false
}

func get(repo map[string]string, repoUrl string) error {
	p("About to git clone/pull repo:", color.FgGreen)
{'fraction_non_alphanumeric': 0.17495987158908508, 'fraction_numerical': 0.022013299701903234, 'mean_word_length': 3.756815703380589, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1926475', 'n_tokens_mistral': 1871, 'n_tokens_neox': 1786, 'n_words': 422}