<commit_before>package main

import (
	"bytes"
	"crypto/sha1"
	"dump/dnav"
	"flag"
	"fmt"
	"github.com/sergi/go-diff/diffmatchpatch"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

var (
	debug        bool
	mChangesFlag bool
	txtFlag      bool

	verbose bool

	yearly  bool
	monthly bool
	daily   bool
	hourly  bool
)

func rdFlags() {
	db := flag.Bool("D", false, "debug flag")

	v := flag.Bool("v", false, "verbose flag")
	c := flag.Bool("c", false, "changes, no diffs flag")
	t := flag.Bool("t", false, "txt flag")

	y := flag.Bool("y", false, "filter yearly")
	m := flag.Bool("m", false, "filter monthly")
	d := flag.Bool("d", false, "filter daily")
	h := flag.Bool("h", true, "filter hourly")
	flag.Parse()

	debug = *db
	dnav.Debug = *db
	mChangesFlag = *c
	txtFlag = *t

	verbose = *v

	yearly = *y
	monthly = *m
	daily = *d
	hourly = *h
}

func Dprintf(format string, a ...interface{}) (n int, err error) {

	if !debug {
		return 0, nil
	}
	return fmt.Fprintf(os.Stderr, "hist: "+format, a...)
}

func usage() {
	log.Fatal("hist [-Dvc] [-ymdh] file_path")
}

type pathDump struct {
	path string
	os.FileInfo
}

func pathsBeforeRec(dDate dnav.DumpDate, root string) (paths []string, err error) {
	files, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}
	found := 0
	for _, f := range files {
		fName := f.Name()
		if _, err := strconv.Atoi(fName); err == nil {
			found++
			files, err := pathsBeforeRec(dDate, root+"/"+fName)
			if err == nil {
				paths = append(paths, files...)
			}
		}
	}
	if found == 0 {

		return []string{root}, nil
	}

	return paths, nil
}

func pathsBefore(dDate dnav.DumpDate, roots dnav.Roots) (paths []string, err error) {
	paths, err = pathsBeforeRec(dDate, roots.DumpRoot)
	ps := paths
	paths = nil
	lastD := dDate
	for _, p := range ps {
		d, err2 := dnav.ParseDumpPath(p, roots)
		if err2 != nil {
			return nil, err2
		}
		if (&d).IsAfter(dDate) {
			continue
		}
		if yearly && d.SameYear(&lastD) {
			continue
		}
		if monthly && d.SameMonth(&lastD) {
			continue
		}
		if daily && d.SameDay(&lastD) {
			continue
		}
		if hourly && d.SameHour(&lastD) {
			continue
		}
		paths = append(paths, p)
		lastD = d
	}
	return paths, nil
}

func fmtDiff(diffs []diffmatchpatch.Diff, curr *File, new *File) string {
	var buff bytes.Buffer
	nlRight := 0
	nlLeft := 0
	Dprintf("\nDIFFs--\n")
	for _, diff := range diffs {
		text := diff.Text
		nlDiff := strings.Count(text, "\n")
		if len(text) == 0 {
			continue
		}
		diffStr := ""
		switch diff.Type {
		case diffmatchpatch.DiffInsert:
			Dprintf(">--insert %d, (%d %d)\n", nlDiff, nlLeft, nlRight)
			buff.WriteString(fmt.Sprintf("\n%s:%d,%d %s:%d,%d\n", curr.path, nlLeft+1, nlLeft+1, new.path, nlRight+1, nlRight+nlDiff+1))
			for i := nlRight; i < nlRight+nlDiff-1; i++ {
				diffStr += ">" + new.lines[i] + "\n"
			}
			if nlDiff == 0 {
				diffStr += "^" + text + "^" + "\n"
				diffStr += ">" + new.lines[nlRight] + "\n" //TODO: SHOULD MERGE continuous runs
			}
			nlRight += nlDiff
		case diffmatchpatch.DiffDelete:
			Dprintf("<--delete %d, (%d %d)\n", nlDiff, nlLeft, nlRight)
			buff.WriteString(fmt.Sprintf("\n%s:%d,%d %s:%d,%d\n", curr.path, nlLeft+1, nlLeft+nlDiff+1, new.path, nlRight+1, nlRight+1))
			for i := nlLeft; i < nlLeft+nlDiff; i++ {
				diffStr += "<" + curr.lines[i] + "\n"
			}
			if nlDiff == 0 {
				diffStr += "^" + text + "^" + "\n"
				diffStr += "<" + curr.lines[nlLeft] + "\n" //TODO: SHOULD MERGE continuous runs
			}
			nlLeft += nlDiff
		case diffmatchpatch.DiffEqual:
			Dprintf("=--match %d, (%d %d)\n", nlDiff, nlLeft, nlRight)
			nlLeft += nlDiff
			nlRight += nlDiff
		}
		if diff.Type != diffmatchpatch.DiffEqual {
			_, _ = buff.WriteString("\n" + diffStr)
		}

	}

	return buff.String()
}

type File struct {
	path  string
	lines []string
	txt   string
	sha   [20]byte
	info  os.FileInfo
}

func (f *File) String() string {
	s := fmt.Sprintf("%s ", f.path)
	if f.info == nil {
		s += " ###bad info###"
		return s
	}
	s += fmt.Sprintf("%d ", f.info.Size())
	s += fmt.Sprintf("%#o ", f.info.Mode())
	s += fmt.Sprintf("%v ", f.info.ModTime())
	if f.info.IsDir() {
		s += fmt.Sprintf(" d")
	} else {
		s += fmt.Sprintf(" f")
	}

	return s
}

func (f *File) isText() bool {
	return !strings.ContainsRune(f.txt, utf8.RuneError)
}

func (f *File) isDir() bool {
	return f.info != nil && f.info.IsDir()
}

func (f *File) readDir() (txt string, err error) {
	files, err := ioutil.ReadDir(f.path)
	if err != nil {
		return "", err
	}
	for _, fi := range files {
		fp := &File{path: f.path + "/" + fi.Name(), info: fi}
		txt += fmt.Sprintf("\t[]\t%s\n", fp)
	}<commit_msg>fix dir diffs in hist<commit_after>package main

import (
	"bytes"
	"crypto/sha1"
	"dump/dnav"
	"flag"
	"fmt"
	"github.com/sergi/go-diff/diffmatchpatch"
	"io/ioutil"
	"log"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"time"
	"unicode/utf8"
)

var (
	debug        bool
	mChangesFlag bool
	txtFlag      bool

	verbose bool

	yearly  bool
	monthly bool
	daily   bool
	hourly  bool
)

func rdFlags() {
	db := flag.Bool("D", false, "debug flag")

	v := flag.Bool("v", false, "verbose flag")
	c := flag.Bool("c", false, "changes, no diffs flag")
	t := flag.Bool("t", false, "txt flag")

	y := flag.Bool("y", false, "filter yearly")
	m := flag.Bool("m", false, "filter monthly")
	d := flag.Bool("d", false, "filter daily")
	h := flag.Bool("h", true, "filter hourly")
	flag.Parse()

	debug = *db
	dnav.Debug = *db
	mChangesFlag = *c
	txtFlag = *t

	verbose = *v

	yearly = *y
	monthly = *m
	daily = *d
	hourly = *h
}

func Dprintf(format string, a ...interface{}) (n int, err error) {

	if !debug {
		return 0, nil
	}
	return fmt.Fprintf(os.Stderr, "hist: "+format, a...)
}

func usage() {
	log.Fatal("hist [-Dvc] [-ymdh] file_path")
}

type pathDump struct {
	path string
	os.FileInfo
}

func pathsBeforeRec(dDate dnav.DumpDate, root string) (paths []string, err error) {
	files, err := ioutil.ReadDir(root)
	if err != nil {
		return nil, err
	}
	found := 0
	for _, f := range files {
		fName := f.Name()
		if _, err := strconv.Atoi(fName); err == nil {
			found++
			files, err := pathsBeforeRec(dDate, root+"/"+fName)
			if err == nil {
				paths = append(paths, files...)
			}
		}
	}
	if found == 0 {

		return []string{root}, nil
	}

	return paths, nil
}

func pathsBefore(dDate dnav.DumpDate, roots dnav.Roots) (paths []string, err error) {
	paths, err = pathsBeforeRec(dDate, roots.DumpRoot)
	ps := paths
	paths = nil
	lastD := dDate
	for _, p := range ps {
		d, err2 := dnav.ParseDumpPath(p, roots)
		if err2 != nil {
			return nil, err2
		}
		if (&d).IsAfter(dDate) {
			continue
		}
		if yearly && d.SameYear(&lastD) {
			continue
		}
		if monthly && d.SameMonth(&lastD) {
			continue
		}
		if daily && d.SameDay(&lastD) {
			continue
		}
		if hourly && d.SameHour(&lastD) {
			continue
		}
		paths = append(paths, p)
		lastD = d
	}
	return paths, nil
}

func fmtDiff(diffs []diffmatchpatch.Diff, curr *File, new *File) string {
	var buff bytes.Buffer
	nlRight := 0
	nlLeft := 0
	Dprintf("\nDIFFs--\n")
	for _, diff := range diffs {
		text := diff.Text
		nlDiff := strings.Count(text, "\n")
		if len(text) == 0 {
			continue
		}
		diffStr := ""
		switch diff.Type {
		case diffmatchpatch.DiffInsert:
			Dprintf(">--insert %d, (%d %d)\n", nlDiff, nlLeft, nlRight)
			buff.WriteString(fmt.Sprintf("\n%s:%d,%d %s:%d,%d\n", curr.path, nlLeft+1, nlLeft+1, new.path, nlRight+1, nlRight+nlDiff+1))
			for i := nlRight; i < nlRight+nlDiff-1; i++ {
				diffStr += ">" + new.lines[i] + "\n"
			}
			if nlDiff == 0 {
				//diffStr += "^" + text + "^" + "\n"
				diffStr += ">" + new.lines[nlRight] + "\n" //TODO: SHOULD MERGE continuous runs
			}
			nlRight += nlDiff
		case diffmatchpatch.DiffDelete:
			Dprintf("<--delete %d, (%d %d)\n", nlDiff, nlLeft, nlRight)
			buff.WriteString(fmt.Sprintf("\n%s:%d,%d %s:%d,%d\n", curr.path, nlLeft+1, nlLeft+nlDiff+1, new.path, nlRight+1, nlRight+1))
			for i := nlLeft; i < nlLeft+nlDiff; i++ {
				diffStr += "<" + curr.lines[i] + "\n"
			}
			if nlDiff == 0 {
				//diffStr += "^" + text + "^" + "\n"
				diffStr += "<" + curr.lines[nlLeft] + "\n" //TODO: SHOULD MERGE continuous runs
			}
			nlLeft += nlDiff
		case diffmatchpatch.DiffEqual:
			Dprintf("=--match %d, (%d %d)\n", nlDiff, nlLeft, nlRight)
			nlLeft += nlDiff
			nlRight += nlDiff
		}
		if diff.Type != diffmatchpatch.DiffEqual {
			_, _ = buff.WriteString("\n" + diffStr)
		}

	}

	return buff.String()
}

type File struct {
	path  string
	lines []string
	txt   string
	sha   [20]byte
	info  os.FileInfo
}

func (f *File) String() string {
	s := fmt.Sprintf("%s ", f.path)
	if f.info == nil {
		s += " ###bad info###"
		return s
	}
	s += fmt.Sprintf("%d ", f.info.Size())
	s += fmt.Sprintf("%#o ", f.info.Mode())
	s += fmt.Sprintf("%v ", f.info.ModTime())
	if f.info.IsDir() {
		s += fmt.Sprintf(" d")
	} else {
		s += fmt.Sprintf(" f")
	}

	return s
}

func (f *File) isText() bool {
	return !strings.ContainsRune(f.txt, utf8.RuneError)
}

func (f *File) isDir() bool {
	return f.info != nil && f.info.IsDir()
}

func (f *File) readDir() (txt string, err error) {
	files, err := ioutil.ReadDir(f.path)
	if err != nil {
		return "", err
	}
	for _, fi := range files {
		fp := &File{path: fi.Name(), info: fi}
		txt += fmt.Sprintf("\t[]\t%s\n", fp)
	}
{'fraction_non_alphanumeric': 0.18902373536677047, 'fraction_numerical': 0.005369992482010525, 'mean_word_length': 3.082419991231916, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6028475', 'n_tokens_mistral': 4073, 'n_tokens_neox': 3811, 'n_words': 943}