<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"log"

	"github.com/dgageot/getme/cache"
	"github.com/dgageot/getme/files"
	"github.com/dgageot/getme/zip"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var authToken string

func main() {
	var rootCmd = &cobra.Command{Use: "getme"}

	rootCmd.PersistentFlags().StringVar(&authToken, "authToken", "", "Api authentication token")

	rootCmd.AddCommand(&cobra.Command{
		Use: "Download",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 1 {
				return errors.New("An url must be provided")
			}
			url := args[0]

			return Download(url)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Copy",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination must be provided")
			}
			url := args[0]
			destination := args[1]

			return Copy(url, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Unzip",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination folder must be provided")
			}
			url := args[0]
			destinationFolder := args[1]

			return Unzip(url, destinationFolder)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "UnzipSingleFile",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 3 {
				return errors.New("An url, a file name and a destination must be provided")
			}
			url := args[0]
			name := args[1]
			destination := args[2]

			return UnzipSingleFile(url, name, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Prune",
		RunE: func(cmd *cobra.Command, args []string) error {
			return Prune()
		},
	})

	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

// Download retrieves an url from the cache or download it if it's absent.
// Then print the path to that file to stdout.
func Download(url string) error {
	// Discard all the logs. We only want to output the path to the file
	log.SetOutput(ioutil.Discard)

	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	fmt.Println(source)

	return nil
}

// Copy retrieves an url from the cache or download it if it's absent.
// Then it copies the file to a destination path.
func Copy(url string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Copy", url, "to", destination)

	return files.Copy(source, destination)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips the file to a destination directory.
func Unzip(url string, destinationDirectory string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", url, "to", destinationDirectory)

	return zip.Extract(source, destinationDirectory)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips a single file from that zip to a destination path.
func UnzipSingleFile(url string, name string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", name, "from", url, "to", destination)

	return zip.ExtractFile(source, name, destination)
}

// Prune prunes the cache.
func Prune() error {
	return cache.Prune()
}

func headers() []string {
	if authToken == "" {
		return nil
	}

	return []string{fmt.Sprintf("Authorization=Bearer%s", authToken)}
}<commit_msg>Fix download with token<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"log"

	"github.com/dgageot/getme/cache"
	"github.com/dgageot/getme/files"
	"github.com/dgageot/getme/zip"
	"github.com/pkg/errors"
	"github.com/spf13/cobra"
)

var authToken string

func main() {
	var rootCmd = &cobra.Command{Use: "getme"}

	rootCmd.PersistentFlags().StringVar(&authToken, "authToken", "", "Api authentication token")

	rootCmd.AddCommand(&cobra.Command{
		Use: "Download",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 1 {
				return errors.New("An url must be provided")
			}
			url := args[0]

			return Download(url)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Copy",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination must be provided")
			}
			url := args[0]
			destination := args[1]

			return Copy(url, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Unzip",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 2 {
				return errors.New("An url and a destination folder must be provided")
			}
			url := args[0]
			destinationFolder := args[1]

			return Unzip(url, destinationFolder)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "UnzipSingleFile",
		RunE: func(cmd *cobra.Command, args []string) error {
			if len(args) < 3 {
				return errors.New("An url, a file name and a destination must be provided")
			}
			url := args[0]
			name := args[1]
			destination := args[2]

			return UnzipSingleFile(url, name, destination)
		},
	})

	rootCmd.AddCommand(&cobra.Command{
		Use: "Prune",
		RunE: func(cmd *cobra.Command, args []string) error {
			return Prune()
		},
	})

	if err := rootCmd.Execute(); err != nil {
		log.Fatal(err)
	}
}

// Download retrieves an url from the cache or download it if it's absent.
// Then print the path to that file to stdout.
func Download(url string) error {
	// Discard all the logs. We only want to output the path to the file
	log.SetOutput(ioutil.Discard)

	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	fmt.Println(source)

	return nil
}

// Copy retrieves an url from the cache or download it if it's absent.
// Then it copies the file to a destination path.
func Copy(url string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Copy", url, "to", destination)

	return files.Copy(source, destination)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips the file to a destination directory.
func Unzip(url string, destinationDirectory string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", url, "to", destinationDirectory)

	return zip.Extract(source, destinationDirectory)
}

// Download retrieves an url from the cache or download it if it's absent.
// Then it unzips a single file from that zip to a destination path.
func UnzipSingleFile(url string, name string, destination string) error {
	source, err := cache.Download(url, headers())
	if err != nil {
		return err
	}

	log.Println("Unzip", name, "from", url, "to", destination)

	return zip.ExtractFile(source, name, destination)
}

// Prune prunes the cache.
func Prune() error {
	return cache.Prune()
}

func headers() []string {
	if authToken == "" {
		return nil
	}

	return []string{fmt.Sprintf("Authorization=Bearer %s", authToken)}
}
{'fraction_non_alphanumeric': 0.13321971310893338, 'fraction_numerical': 0.0039767078539980115, 'mean_word_length': 3.7293485560779045, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 29, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2900248', 'n_tokens_mistral': 2588, 'n_tokens_neox': 2376, 'n_words': 740}