<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/mibk/dupl/job"
	"github.com/mibk/dupl/output"
	"github.com/mibk/dupl/remote"
	"github.com/mibk/dupl/syntax"
)

const DefaultThreshold = 15

var (
	dir        = "."
	verbose    = flag.Bool("verbose", false, "explain what is being done")
	threshold  = flag.Int("threshold", DefaultThreshold, "minimum token sequence as a clone")
	serverPort = flag.String("serve", "", "run server at port")
	addrs      AddrList
	files      = flag.Bool("files", false, "files names from stdin")

	html     = flag.Bool("html", false, "html output")
	plumbing = flag.Bool("plumbing", false, "plumbing output for consumption by scripts or tools")
)

type AddrList []string

func (l *AddrList) String() string {
	return fmt.Sprintf("%v", *l)
}

func (l *AddrList) Set(val string) error {
	*l = append(*l, val)
	return nil
}

func init() {
	flag.BoolVar(verbose, "v", false, "alias for -verbose")
	flag.IntVar(threshold, "t", DefaultThreshold, "alias for -threshold")
	flag.Var(&addrs, "connect", "connect to the given 'addr:port'")
	flag.Var(&addrs, "c", "alias for -connect")
}

func main() {
	flag.Parse()
	if *html && *plumbing {
		log.Fatal("you can have either plumbing or HTML output")
	}
	if flag.NArg() > 0 {
		dir = flag.Arg(0)
	}

	if len(addrs) != 0 {
		schan := job.Parse(FilesFeed())
		nodesChan := remote.RunClient(addrs, *threshold, schan, *verbose)
		printDupls(nodesChan)
	} else if *serverPort != "" {
		remote.RunServer(*serverPort)
	} else {
		if *verbose {
			log.Println("Building suffix tree")
		}
		schan := job.Parse(FilesFeed())
		t, data, done := job.BuildTree(schan)
		<-done

		// finish stream
		t.Update(&syntax.Node{Type: -1})

		if *verbose {
			log.Println("Searching for clones")
		}
		mchan := t.FindDuplOver(*threshold)
		duplChan := make(chan syntax.Match)
		go func() {
			for m := range mchan {
				match := syntax.FindSyntaxUnits(*data, m, *threshold)
				if len(match.Frags) > 0 {
					duplChan <- match
				}
			}
			close(duplChan)
		}()
		printDupls(duplChan)
	}
}

func FilesFeed() chan string {
	if *files {
		fchan := make(chan string)
		go func() {
			s := bufio.NewScanner(os.Stdin)
			for s.Scan() {
				f := s.Text()
				if strings.HasPrefix(f, "./") {
					f = f[2:]
				}
				fchan <- f
			}
			close(fchan)
		}()
		return fchan
	}
	return CrawlDir(dir)
}

func CrawlDir(dir string) chan string {
	// collect files
	fchan := make(chan string)
	go func() {
		filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
			if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") {
				fchan <- path
			}
			return nil
		})
		close(fchan)
	}()
	return fchan
}

type LocalFileReader struct{}

func (r *LocalFileReader) ReadFile(node *syntax.Node) ([]byte, error) {
	return ioutil.ReadFile(node.Filename)
}<commit_msg>dupl: read paths from command line arguments<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strings"

	"github.com/mibk/dupl/job"
	"github.com/mibk/dupl/output"
	"github.com/mibk/dupl/remote"
	"github.com/mibk/dupl/syntax"
)

const DefaultThreshold = 15

var (
	paths      = []string{"."}
	verbose    = flag.Bool("verbose", false, "explain what is being done")
	threshold  = flag.Int("threshold", DefaultThreshold, "minimum token sequence as a clone")
	serverPort = flag.String("serve", "", "run server at port")
	addrs      AddrList
	files      = flag.Bool("files", false, "files names from stdin")

	html     = flag.Bool("html", false, "html output")
	plumbing = flag.Bool("plumbing", false, "plumbing output for consumption by scripts or tools")
)

type AddrList []string

func (l *AddrList) String() string {
	return fmt.Sprintf("%v", *l)
}

func (l *AddrList) Set(val string) error {
	*l = append(*l, val)
	return nil
}

func init() {
	flag.BoolVar(verbose, "v", false, "alias for -verbose")
	flag.IntVar(threshold, "t", DefaultThreshold, "alias for -threshold")
	flag.Var(&addrs, "connect", "connect to the given 'addr:port'")
	flag.Var(&addrs, "c", "alias for -connect")
}

func main() {
	flag.Parse()
	if *html && *plumbing {
		log.Fatal("you can have either plumbing or HTML output")
	}
	if flag.NArg() > 0 {
		paths = flag.Args()
	}

	if len(addrs) != 0 {
		schan := job.Parse(FilesFeed())
		nodesChan := remote.RunClient(addrs, *threshold, schan, *verbose)
		printDupls(nodesChan)
	} else if *serverPort != "" {
		remote.RunServer(*serverPort)
	} else {
		if *verbose {
			log.Println("Building suffix tree")
		}
		schan := job.Parse(FilesFeed())
		t, data, done := job.BuildTree(schan)
		<-done

		// finish stream
		t.Update(&syntax.Node{Type: -1})

		if *verbose {
			log.Println("Searching for clones")
		}
		mchan := t.FindDuplOver(*threshold)
		duplChan := make(chan syntax.Match)
		go func() {
			for m := range mchan {
				match := syntax.FindSyntaxUnits(*data, m, *threshold)
				if len(match.Frags) > 0 {
					duplChan <- match
				}
			}
			close(duplChan)
		}()
		printDupls(duplChan)
	}
}

func FilesFeed() chan string {
	if *files {
		fchan := make(chan string)
		go func() {
			s := bufio.NewScanner(os.Stdin)
			for s.Scan() {
				f := s.Text()
				if strings.HasPrefix(f, "./") {
					f = f[2:]
				}
				fchan <- f
			}
			close(fchan)
		}()
		return fchan
	}
	return CrawlPaths(paths)
}

func CrawlPaths(paths []string) chan string {
	fchan := make(chan string)
	go func() {
		for _, path := range paths {
			info, err := os.Lstat(path)
			if err != nil {
				panic(err)
			}
			if !info.IsDir() {
				fchan <- path
				continue
			}
			filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
				if !info.IsDir() && strings.HasSuffix(info.Name(), ".go") {
					fchan <- path
				}
				return nil
			})
		}
		close(fchan)
	}()
	return fchan
}

type LocalFileReader struct{}

func (r *LocalFileReader) ReadFile(node *syntax.Node) ([]byte, error) {
	return ioutil.ReadFile(node.Filename)
}
{'fraction_non_alphanumeric': 0.1617917432726057, 'fraction_numerical': 0.0025071034598027745, 'mean_word_length': 3.393538913362702, 'pattern_counts': {'":': 0, '<': 12, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '838308', 'n_tokens_mistral': 2487, 'n_tokens_neox': 2228, 'n_words': 551}