<commit_before>package dup

import (
	"encoding/xml"
	"fmt"
	"io"
	"strings"
	"time"

	ledgertools "github.com/ginabythebay/ledger-tools"
)

const (
	suppressAmountDuplicates = "SuppressAmountDuplicates:"
	suppressCodeDuplicates   = "SuppressCodeDuplicates:"
)

type key struct {
	account string
	amount  string
	date    string
}

func newKey(account, amount string, t time.Time) key {
	return key{account, amount, t.Format("2006/01/02")}
}

func suppressedDates(suppressPrefix string, notes []string) []string {
	var dates []string
	for _, line := range notes {
		split := strings.SplitAfterN(line, suppressPrefix, 2)
		if len(split) != 2 {
			continue
		}
		someCandidates := strings.Split(split[1], ",")
		for _, c := range someCandidates {
			c = strings.TrimSpace(c)
			if len(c) < 10 {
				// dates are 10 characters long.
				break
			}
			_, err := time.Parse("2006/01/02", c[:10])
			if err == nil {
				dates = append(dates, c[:10])
			}
			if len(c) > 10 {
				// sometimes there is glop after a valid date.  If we hit that case, just bail out until the next line
				break
			}
		}
	}

	return dates
}

func isDateSuppressed(date string, suppressPrefix string, notes []string) bool {
	for _, d := range suppressedDates(suppressPrefix, notes) {
		if date == d {
			return true
		}
	}
	return false
}

type duplicate interface {
	compilerText() (string, error)
	isSuppressed() bool
	accumXMLErrors(accum map[string]*file)
}

// Finder tracks postings and looks for potential duplicates, based on the amount and the date.
type Finder struct {
	// # days to use when looking for matches.  0 means only look for matches on exactly the same day
	Days int

	m map[key][]*ledgertools.Posting

	allDuplicates []duplicate
}

// NewFinder creates a new Finder.
func NewFinder(days int) *Finder {
	return &Finder{
		Days: days,
		m:    make(map[key][]*ledgertools.Posting),
	}
}

// Add adds t and its postings and tracks any existing postings that
// have the same amount and are within the configured number of days.
func (f *Finder) Add(t *ledgertools.Transaction) {
	for _, p := range t.Postings {
		f.addPosting(p)
	}
}

func (f *Finder) addPosting(p *ledgertools.Posting) {
	var matches []*ledgertools.Posting
	t := p.Xact.Date
	amount := p.AmountText()
	k := newKey(p.Account, amount, t)

	if f.Days >= 0 {
		matches = append(matches, f.m[k]...)
	}
	for i := 1; i <= f.Days; i++ {
		before := t.AddDate(0, 0, -i)
		matches = append(matches, f.m[newKey(p.Account, amount, before)]...)

		after := t.AddDate(0, 0, i)
		matches = append(matches, f.m[newKey(p.Account, amount, after)]...)
	}

	for _, m := range matches {
		p := amountPair{m, p}
		if !p.isSuppressed() {
			f.allDuplicates = append(f.allDuplicates, p)
		}
	}

	f.m[k] = append(f.m[k], p)
}

// WriteJavacStyle writes javac-style output for all duplicates found.
func (f *Finder) WriteJavacStyle(w io.Writer) error {
	matchCount := 0
	for _, d := range f.allDuplicates {<commit_msg>refactoring<commit_after>package dup

import (
	"encoding/xml"
	"fmt"
	"io"
	"strings"
	"time"

	ledgertools "github.com/ginabythebay/ledger-tools"
)

const (
	suppressAmountDuplicates = "SuppressAmountDuplicates:"
	suppressCodeDuplicates   = "SuppressCodeDuplicates:"
)

type amountKey struct {
	account string
	amount  string
	date    string
}

func newAmountKey(account, amount string, t time.Time) amountKey {
	return amountKey{account, amount, t.Format("2006/01/02")}
}

func suppressedDates(suppressPrefix string, notes []string) []string {
	var dates []string
	for _, line := range notes {
		split := strings.SplitAfterN(line, suppressPrefix, 2)
		if len(split) != 2 {
			continue
		}
		someCandidates := strings.Split(split[1], ",")
		for _, c := range someCandidates {
			c = strings.TrimSpace(c)
			if len(c) < 10 {
				// dates are 10 characters long.
				break
			}
			_, err := time.Parse("2006/01/02", c[:10])
			if err == nil {
				dates = append(dates, c[:10])
			}
			if len(c) > 10 {
				// sometimes there is glop after a valid date.  If we hit that case, just bail out until the next line
				break
			}
		}
	}

	return dates
}

func isDateSuppressed(date string, suppressPrefix string, notes []string) bool {
	for _, d := range suppressedDates(suppressPrefix, notes) {
		if date == d {
			return true
		}
	}
	return false
}

type duplicate interface {
	compilerText() (string, error)
	isSuppressed() bool
	accumXMLErrors(accum map[string]*file)
}

// Finder tracks postings and looks for potential duplicates, based on the amount and the date.
type Finder struct {
	// # days to use when looking for matches.  0 means only look for matches on exactly the same day
	Days int

	amountMap map[amountKey][]*ledgertools.Posting

	allDuplicates []duplicate
}

// NewFinder creates a new Finder.
func NewFinder(days int) *Finder {
	return &Finder{
		Days:      days,
		amountMap: make(map[amountKey][]*ledgertools.Posting),
	}
}

// Add adds t and its postings and tracks any existing postings that
// have the same amount and are within the configured number of days.
func (f *Finder) Add(t *ledgertools.Transaction) {
	for _, p := range t.Postings {
		f.addAmountPosting(p)
	}
}

func (f *Finder) addAmountPosting(p *ledgertools.Posting) {
	var matches []*ledgertools.Posting
	t := p.Xact.Date
	amount := p.AmountText()
	k := newAmountKey(p.Account, amount, t)

	if f.Days >= 0 {
		matches = append(matches, f.amountMap[k]...)
	}
	for i := 1; i <= f.Days; i++ {
		before := t.AddDate(0, 0, -i)
		matches = append(matches, f.amountMap[newAmountKey(p.Account, amount, before)]...)

		after := t.AddDate(0, 0, i)
		matches = append(matches, f.amountMap[newAmountKey(p.Account, amount, after)]...)
	}

	for _, m := range matches {
		p := amountPair{m, p}
		if !p.isSuppressed() {
			f.allDuplicates = append(f.allDuplicates, p)
		}
	}

	f.amountMap[k] = append(f.amountMap[k], p)
}

// WriteJavacStyle writes javac-style output for all duplicates found.
func (f *Finder) WriteJavacStyle(w io.Writer) error {
	matchCount := 0
	for _, d := range f.allDuplicates {
{'fraction_non_alphanumeric': 0.1298140770252324, 'fraction_numerical': 0.012284196547144754, 'mean_word_length': 3.6168582375478926, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 2}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2161332', 'n_tokens_mistral': 2313, 'n_tokens_neox': 2130, 'n_words': 620}