<commit_before>package main

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"fmt"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
	"os/exec"
	fpath "path/filepath"
	"strings"
)

var tick, cross string = `✔`, `✖`

func verifyChecksum(c *cli.Context) {
	fileInfo, filepath := checkFilepathArgument(c)
	if fileInfo == nil {
		return
	}

	if c.GlobalBool("delete") {
		fmt.Println(`"delete" doesn't apply to this commmand`)
		return
	}

	mode := "batch"
	if c.GlobalBool("single") {
		mode = "single"
	}

	fmt.Printf("The following filepath (%s mode) will be processed: %s\n", mode, filepath)
	notifyDeleteMode(c)

	if !shouldContinue(c) {
		return
	}

	// We need to get the working directory
	// so that we can get the path to metaflac
	workingDirectory, err := os.Getwd()
	if err != nil {
		fmt.Println("could not get working directory for some reason")
		fmt.Println("reason is: " + err.Error())
		fmt.Println("aborting!")
		return
	}

	if mode == "single" {
		verifyProcessPath(filepath, fileInfo.Name(), workingDirectory)
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() {
			verifyProcessPath(fpath.Join(filepath, file.Name()), file.Name(), workingDirectory)
		}
	}
}

func verifyProcessPath(directory string, name string, workingDirectory string) {
	// Let us know what is currently being processed
	fmt.Print(directory + "... ")

	baseFilename := fpath.Join(directory, name+".")
	ffpFilename := baseFilename + "ffp"
	md5Filename := baseFilename + "md5"

	_, ffpErr := os.Stat(ffpFilename)
	_, md5Err := os.Stat(md5Filename)
	for _, err := range []error{ffpErr, md5Err} {
		if err != nil {
			file := "ffp"
			if err == md5Err {
				file = "md5"
			}
			fmt.Printf("\n> %s read error: (%s)", file, getFileErrorReason(err))
		}
	}

	md5Success, md5ReadError := false, false
	if md5Err == nil {
		md5Success, md5ReadError = verifyMD5(md5Filename, directory)
	}

	ffpSuccess := false<commit_msg>verify now ignores the __wikifiles folder<commit_after>package main

import (
	"bufio"
	"bytes"
	"crypto/md5"
	"fmt"
	"github.com/codegangsta/cli"
	"io/ioutil"
	"os"
	"os/exec"
	fpath "path/filepath"
	"strings"
)

var tick, cross string = `✔`, `✖`

func verifyChecksum(c *cli.Context) {
	fileInfo, filepath := checkFilepathArgument(c)
	if fileInfo == nil {
		return
	}

	if c.GlobalBool("delete") {
		fmt.Println(`"delete" doesn't apply to this commmand`)
		return
	}

	mode := "batch"
	if c.GlobalBool("single") {
		mode = "single"
	}

	fmt.Printf("The following filepath (%s mode) will be processed: %s\n", mode, filepath)
	notifyDeleteMode(c)

	if !shouldContinue(c) {
		return
	}

	// We need to get the working directory
	// so that we can get the path to metaflac
	workingDirectory, err := os.Getwd()
	if err != nil {
		fmt.Println("could not get working directory for some reason")
		fmt.Println("reason is: " + err.Error())
		fmt.Println("aborting!")
		return
	}

	if mode == "single" {
		verifyProcessPath(filepath, fileInfo.Name(), workingDirectory)
		return
	}

	files, _ := ioutil.ReadDir(filepath)
	for _, file := range files {
		if file.IsDir() && file.Name() != "__wikifiles" {
			verifyProcessPath(fpath.Join(filepath, file.Name()), file.Name(), workingDirectory)
		}
	}
}

func verifyProcessPath(directory string, name string, workingDirectory string) {
	// Let us know what is currently being processed
	fmt.Print(directory + "... ")

	baseFilename := fpath.Join(directory, name+".")
	ffpFilename := baseFilename + "ffp"
	md5Filename := baseFilename + "md5"

	_, ffpErr := os.Stat(ffpFilename)
	_, md5Err := os.Stat(md5Filename)
	for _, err := range []error{ffpErr, md5Err} {
		if err != nil {
			file := "ffp"
			if err == md5Err {
				file = "md5"
			}
			fmt.Printf("\n> %s read error: (%s)", file, getFileErrorReason(err))
		}
	}

	md5Success, md5ReadError := false, false
	if md5Err == nil {
		md5Success, md5ReadError = verifyMD5(md5Filename, directory)
	}

	ffpSuccess := false
{'fraction_non_alphanumeric': 0.14781947261663286, 'fraction_numerical': 0.007606490872210953, 'mean_word_length': 3.912826899128269, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1107602', 'n_tokens_mistral': 1513, 'n_tokens_neox': 1436, 'n_words': 362}