<commit_before>// "gitbackup" holds the functions that do the actual backing up of git
// repositories.
package gitbackup

import(
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target map[string]string, backupDirectory string) error {
	log.Printf(`Backing up target "%s"`, target["name"])

	// TODO: replace with a factory pattern?
	switch target["source"] {
	case "github":
		return backupGitHub(target, backupDirectory)
	case "bitbucket":
		return backupBitBucket(target, backupDirectory)
	default:
		return fmt.Errorf(`"%s" is not a recognized source type`, target["source"])
	}
}

// backupGitHub finds all the repositories under a given user or organization
// before backing up each one.
func backupGitHub(target map[string]string, backupDirectory string) error {
	// Create URL to request list of repos.
	var requestURL string = fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target["type"],
		target["entity"],
		target["token"],
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var dat []map[string]interface{}
	if err := json.Unmarshal(contents, &dat); err != nil {
		return fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Back up each repository.
	for _, repo := range dat {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target["entity"], target["token"]),
			1,
		)
		backupRepository(
			target["name"],
			repoName,
			cloneURL,
			backupDirectory,
		)
	}

	// No errors.
	return nil
}

func backupBitBucket(target map[string]string, backupDirectory string) error {
	// TODO: implement.<commit_msg>Implement BitBucket backups.<commit_after>// "gitbackup" holds the functions that do the actual backing up of git
// repositories.
package gitbackup

import(
	"encoding/json"
	"fmt"
	"io/ioutil"
	"log"
	"net/http"
	"os"
	"os/exec"
	"path/filepath"
	"strings"
)

// BackupTarget backs up an entity that holds one or more git repositories and
// has an interface to retrieve that list of repositories.
// Examples of entities include:
//   - A GitHub user.
//   - A BitBucket user.
//   - A GitHub organization.
func BackupTarget(target map[string]string, backupDirectory string) error {
	log.Printf(`Backing up target "%s"`, target["name"])

	// TODO: replace with a factory pattern?
	switch target["source"] {
	case "github":
		return backupGitHub(target, backupDirectory)
	case "bitbucket":
		return backupBitBucket(target, backupDirectory)
	default:
		return fmt.Errorf(`"%s" is not a recognized source type`, target["source"])
	}
}

// backupGitHub finds all the repositories under a given user or organization
// before backing up each one.
func backupGitHub(target map[string]string, backupDirectory string) error {
	// Create URL to request list of repos.
	var requestURL string = fmt.Sprintf(
		"https://api.github.com/%s/%s/repos?access_token=%s&per_page=200",
		target["type"],
		target["entity"],
		target["token"],
	)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var dat []map[string]interface{}
	if err := json.Unmarshal(contents, &dat); err != nil {
		return fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Back up each repository.
	for _, repo := range dat {
		repoName, _ := repo["name"].(string)
		cloneURL, _ := repo["clone_url"].(string)
		cloneURL = strings.Replace(
			cloneURL,
			"https://",
			fmt.Sprintf("https://%s:%s@", target["entity"], target["token"]),
			1,
		)
		backupRepository(
			target["name"],
			repoName,
			cloneURL,
			backupDirectory,
		)
	}

	// No errors.
	return nil
}

func backupBitBucket(target map[string]string, backupDirectory string) error {
	// Create URL to request list of repos.
	// TODO: support pagination.
	var requestURL string = fmt.Sprintf(
		"https://%s:%s@bitbucket.org/api/2.0/repositories/%s?page=1&pagelen=100",
		target["entity"],
		target["password"],
		target["entity"],
	)
	fmt.Println(requestURL)

	// Retrieve list of repositories.
	response, err := http.Get(requestURL)
	if err != nil {
		return fmt.Errorf("Failed to connect with the source to retrieve the list of repositories: %s", err)
	}
	defer response.Body.Close()
	contents, err := ioutil.ReadAll(response.Body)
	if err != nil {
		return fmt.Errorf("Failed to retrieve the list of repositories: %s", err)
	}

	// Parse JSON response.
	var metadata map[string]json.RawMessage
	if err := json.Unmarshal(contents, &metadata); err != nil {
		return fmt.Errorf("Failed to parse JSON: %s", err)
	}
	var data []map[string]json.RawMessage
	if err := json.Unmarshal(metadata["values"], &data); err != nil {
		return fmt.Errorf("Failed to parse JSON: %s", err)
	}

	// Back up each repository.
	for _, repo := range data {
		// Parse the remaining JSON message that pertains to this repository.
		var repoName string
		if err := json.Unmarshal(repo["name"], &repoName); err != nil {
			return fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var links map[string]json.RawMessage
		if err := json.Unmarshal(repo["links"], &links); err != nil {
			return fmt.Errorf("Failed to parse JSON: %s", err)
		}
		var cloneLinks []map[string]string
		if err := json.Unmarshal(links["clone"], &cloneLinks); err != nil {
			return fmt.Errorf("Failed to parse JSON: %s", err)
		}

		// Find the https URL to use for cloning.
		var cloneURL string
		for _, link := range cloneLinks {
			if link["name"] == "https" {
				cloneURL = link["href"]
			}
		}
		if cloneURL == "" {
			return fmt.Errorf("Could not determine HTTPS cloning URL: %s", cloneLinks)
		}

		// Back up the repository.
		cloneURL = strings.Replace(
			cloneURL,
			fmt.Sprintf("https://%s@", target["entity"]),
			fmt.Sprintf("https://%s:%s@", target["entity"], target["password"]),
			1,
		)
		backupRepository(
			target["name"],
			repoName,
			cloneURL,
			backupDirectory,
		)
	}
{'fraction_non_alphanumeric': 0.14181818181818182, 'fraction_numerical': 0.002181818181818182, 'mean_word_length': 3.869688385269122, 'pattern_counts': {'":': 4, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 9, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 1, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1491212', 'n_tokens_mistral': 2358, 'n_tokens_neox': 2157, 'n_words': 683}