<commit_before>package main

import (
	"fmt"
	"io/ioutil"
	"path"
	"strings"
	"text/template"
	"time"

	"gopkg.in/urfave/cli.v1"
)

func generateInformation(c *cli.Context) {
	fileInfo, filepath := checkFilepathArgument(c)
	if fileInfo == nil {
		return
	}

	tourName := c.String("tour")
	if tourName == "" {
		cli.ShowSubcommandHelp(c)
		return
	}

	mode := "batch"
	if c.GlobalBool("single") {
		mode = "single"
	}

	tourfile := c.String("tour-file")
	if tourfile != "" {
		fileInfo, tourfileClean := getFileOfType(tourfile, false, "tour-file")
		if fileInfo == nil {
			return
		}
		tourfile = tourfileClean
		fmt.Println("Processing tours from:", tourfile)
	}

	fmt.Println("The current tour is:", tourName)
	fmt.Printf("The following filepath (%s mode) will be processed: %s\n", mode, filepath)
	notifyDeleteMode(c)

	if !shouldContinue(c) {
		return
	}

	tour := new(Tour)
	tour.Name = tourName
	if tourfile != "" { // tourFile is only for reading "alternate vocalists" into tracks map
		if err := getTourFromTourFile(tourfile, tour); err != nil {
			fmt.Println("[Error]", err)
			if !shouldContinue(c) {
				return
			}
		}
	}

	// Stupid windows
	informationTemplate = strings.Replace(informationTemplate, "\n", "\r\n", -1)

	if mode == "single" {
		generateFile(filepath, fileInfo.Name(), *tour, c.GlobalBool("delete"))
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() {
			name := file.Name()
			generateFile(path.Join(filepath, name), name, *tour, c.GlobalBool("delete"))
		}
	}
}

func generateFile(filepath string, name string, tour Tour, deleteMode bool) {
	outputFilename := path.Join(filepath, name+".txt")
	if deleteMode {
		removeFile(outputFilename, true)
		return
	}

	album := new(AlbumData)
	album.Tour = tour.Name

	var useCDNames bool
	var folders []string
	var extraFolders []string
	var files []string

	directoryContents, _ := ioutil.ReadDir(filepath)
	for _, fileinfo := range directoryContents {
		filename := fileinfo.Name()
		isDir := fileinfo.IsDir()
		if isDir {
			if strings.HasPrefix(filename, "CD") {
				folders = append(folders, filename)
				useCDNames = true
			} else {
				extraFolders = append(extraFolders, filename)<commit_msg>don't generate anything for the wikifiles folder<commit_after>package main

import (
	"fmt"
	"io/ioutil"
	"path"
	"strings"
	"text/template"
	"time"

	"gopkg.in/urfave/cli.v1"
)

func generateInformation(c *cli.Context) {
	fileInfo, filepath := checkFilepathArgument(c)
	if fileInfo == nil {
		return
	}

	tourName := c.String("tour")
	if tourName == "" {
		cli.ShowSubcommandHelp(c)
		return
	}

	mode := "batch"
	if c.GlobalBool("single") {
		mode = "single"
	}

	tourfile := c.String("tour-file")
	if tourfile != "" {
		fileInfo, tourfileClean := getFileOfType(tourfile, false, "tour-file")
		if fileInfo == nil {
			return
		}
		tourfile = tourfileClean
		fmt.Println("Processing tours from:", tourfile)
	}

	fmt.Println("The current tour is:", tourName)
	fmt.Printf("The following filepath (%s mode) will be processed: %s\n", mode, filepath)
	notifyDeleteMode(c)

	if !shouldContinue(c) {
		return
	}

	tour := new(Tour)
	tour.Name = tourName
	if tourfile != "" { // tourFile is only for reading "alternate vocalists" into tracks map
		if err := getTourFromTourFile(tourfile, tour); err != nil {
			fmt.Println("[Error]", err)
			if !shouldContinue(c) {
				return
			}
		}
	}

	// Stupid windows
	informationTemplate = strings.Replace(informationTemplate, "\n", "\r\n", -1)

	if mode == "single" {
		generateFile(filepath, fileInfo.Name(), *tour, c.GlobalBool("delete"))
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() && (file.Name() != "__wikifiles") {
			name := file.Name()
			generateFile(path.Join(filepath, name), name, *tour, c.GlobalBool("delete"))
		}
	}
}

func generateFile(filepath string, name string, tour Tour, deleteMode bool) {
	outputFilename := path.Join(filepath, name+".txt")
	if deleteMode {
		removeFile(outputFilename, true)
		return
	}

	album := new(AlbumData)
	album.Tour = tour.Name

	var useCDNames bool
	var folders []string
	var extraFolders []string
	var files []string

	directoryContents, _ := ioutil.ReadDir(filepath)
	for _, fileinfo := range directoryContents {
		filename := fileinfo.Name()
		isDir := fileinfo.IsDir()
		if isDir {
			if strings.HasPrefix(filename, "CD") {
				folders = append(folders, filename)
				useCDNames = true
			} else {
				extraFolders = append(extraFolders, filename)
{'fraction_non_alphanumeric': 0.14345804737657739, 'fraction_numerical': 0.0008855435023245517, 'mean_word_length': 3.986754966887417, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2321322', 'n_tokens_mistral': 1742, 'n_tokens_neox': 1580, 'n_words': 377}