<commit_before>package main

import (
	"bufio"
	"fmt"
	"github.com/cenkalti/log"
	"github.com/cheggaaa/pb"
	"github.com/codegangsta/cli"
	dupes "github.com/danmarg/undupes/libdupes"
	"github.com/dustin/go-humanize"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Horrible hack to convert -v to log Levels.
var orderedLevels []log.Level = []log.Level{log.DEBUG, log.INFO, log.NOTICE, log.WARNING, log.ERROR, log.CRITICAL}

func setLogLevel(l int) error {
	if l >= 0 && l < len(orderedLevels) {
		log.SetLevel(orderedLevels[l])
		return nil
	}
	return fmt.Errorf("invalid log level specified")
}

func main() {
	app := cli.NewApp()
	app.Name = "undupes"
	app.Usage = "manage duplicate files"
	app.Author = "<NAME>"
	app.Email = "<EMAIL>"
	app.Version = "0.1"
	app.Commands = []cli.Command{
		{
			Name:      "interactive",
			ShortName: "i",
			Usage:     "interactive mode",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				cli.IntFlag{
					Name:  "v",
					Usage: "log level",
					Value: 3, // Don't print as much in interactive mode.
				},
			},
			Action: func(c *cli.Context) {
				if err := setLogLevel(c.Int("v")); err != nil {
					fmt.Println(err)
					return
				}
				if err := runInteractive(c.Bool("dry_run"), c.String("directory")); err != nil {
					fmt.Println(err)
				}
			},
		},
		{
			Name:      "print",
			ShortName: "p",
			Usage:     "print duplicates",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				cli.StringFlag{
					Name:  "output, o",
					Usage: "output file",
				},
			},
			Action: func(c *cli.Context) {
				if err := runPrint(c.String("directory"), c.String("output")); err != nil {
					fmt.Println(err)
				}
			},
		},
		{
			Name:      "auto",
			ShortName: "a",
			Usage:     "automatically resolve duplicates",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				cli.StringFlag{
					Name:  "prefer, p",
					Usage: "prefer to keep files matching this pattern",
				},
				cli.StringFlag{
					Name:  "over, o",
					Usage: "used with --prefer; will restrict preferred files to those where the duplicate matches --over",
				},
				cli.BoolFlag{
					Name:  "invert, i",
					Usage: "invert matching logic; preferred files with be prioritized for deletion rather than retention",
				},
				cli.BoolFlag{
					Name:  "dry_run",<commit_msg>Check for required args.<commit_after>package main

import (
	"bufio"
	"fmt"
	"github.com/cenkalti/log"
	"github.com/cheggaaa/pb"
	"github.com/codegangsta/cli"
	dupes "github.com/danmarg/undupes/libdupes"
	"github.com/dustin/go-humanize"
	"os"
	"regexp"
	"strconv"
	"strings"
)

// Horrible hack to convert -v to log Levels.
var orderedLevels []log.Level = []log.Level{log.DEBUG, log.INFO, log.NOTICE, log.WARNING, log.ERROR, log.CRITICAL}

func setLogLevel(l int) error {
	if l >= 0 && l < len(orderedLevels) {
		log.SetLevel(orderedLevels[l])
		return nil
	}
	return fmt.Errorf("invalid log level specified")
}

func main() {
	app := cli.NewApp()
	app.Name = "undupes"
	app.Usage = "manage duplicate files"
	app.Author = "<NAME>"
	app.Email = "<EMAIL>"
	app.Version = "0.1"
	app.Commands = []cli.Command{
		{
			Name:      "interactive",
			ShortName: "i",
			Usage:     "interactive mode",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				cli.IntFlag{
					Name:  "v",
					Usage: "log level",
					Value: 3, // Don't print as much in interactive mode.
				},
			},
			Action: func(c *cli.Context) {
				if err := setLogLevel(c.Int("v")); err != nil {
					fmt.Println(err)
					return
				}
				if err := runInteractive(c.Bool("dry_run"), c.String("directory")); err != nil {
					fmt.Println(err)
				}
			},
		},
		{
			Name:      "print",
			ShortName: "p",
			Usage:     "print duplicates",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				cli.StringFlag{
					Name:  "output, o",
					Usage: "output file",
				},
			},
			Action: func(c *cli.Context) {
				if c.String("directory") == "" {
					fmt.Println("--directory required")
					return
				}
				if c.String("output") == "" {
					fmt.Println("--output required")
					return
				}
				if err := runPrint(c.String("directory"), c.String("output")); err != nil {
					fmt.Println(err)
				}
			},
		},
		{
			Name:      "auto",
			ShortName: "a",
			Usage:     "automatically resolve duplicates",
			Flags: []cli.Flag{
				cli.StringFlag{
					Name:  "directory, d",
					Usage: "directory in which to find duplicates (required)",
				},
				cli.StringFlag{
					Name:  "prefer, p",
					Usage: "prefer to keep files matching this pattern",
				},
				cli.StringFlag{
					Name:  "over, o",
					Usage: "used with --prefer; will restrict preferred files to those where the duplicate matches --over",
				},
				cli.BoolFlag{
					Name:  "invert, i",
					Usage: "invert matching logic; preferred files with be prioritized for deletion rather than retention",
				},
				cli.BoolFlag{
					Name:  "dry_run",
{'fraction_non_alphanumeric': 0.15625586193959856, 'fraction_numerical': 0.0015006565372350403, 'mean_word_length': 2.726065688329839, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1800934', 'n_tokens_mistral': 2342, 'n_tokens_neox': 1865, 'n_words': 476}