<commit_before>package main

import (
	"bufio"
	"crypto"
	"encoding/hex"
	"golang.org/x/exp/slices"
	"io"
	"os"
	"strings"
)

// Choose the algorithm with the longest digest size
func bestHash(checksums []*Checksum, ignore crypto.Hash) *Checksum {
	max := 0
	var best *Checksum
	for _, checksum := range checksums {
		if ignore != checksum.hash && checksum.hash.Size() > max {
			max = checksum.hash.Size()
			best = checksum
		}
	}
	return best
}

// Choose the best algorithm, or 2 if one of them is insecure
func bestHashes(checksums []*Checksum) (best []*Checksum) {
	best1 := bestHash(checksums, 0)
	if best1 == nil {
		return nil
	}
	best = append(best, best1)
	// Return 2 algorithms if the "best" of them is insecure
	if slices.Contains(insecure, best1.hash) {
		best2 := bestHash(checksums, best1.hash)
		if best2 != nil {
			best = append(best, best2)
		}
	}
	return best
}

func isChosen(h crypto.Hash) bool {
	for i := range chosen {
		if chosen[i].hash == h {
			return true
		}
	}
	return false
}

func parseLine(line string, lineno uint64) *Checksums {
	var algorithm, file, digest string
	var hash crypto.Hash
	var match []string
	var sum []byte
	var err error
	var ok bool

	if !opts.bsd {
		match = regex.gnu.FindStringSubmatch(line)
		if match != nil {
			digest, file = strings.ToLower(match[1]), unescapeFilename(match[2])
			if len(chosen) == 1 {
				algorithm = algorithms[chosen[0].hash].name
			} else {
				algorithm = size2hash[len(digest)]
			}
		}
	}
	if !opts.gnu {
		match = regex.bsd.FindStringSubmatch(line)
		if match != nil {
			algorithm, file, digest = match[1], match[2], strings.ToLower(match[3])
		}
	}
	sum, err = hex.DecodeString(digest)
	if hash, ok = name2Hash[algorithm]; !ok || err != nil || len(chosen) > 0 && !isChosen(hash) {
		stats.invalid++
		if opts.warn {
			logger.Printf("Invalid digest at line %d\n", lineno)
		} else if opts.strict {
			logger.Fatalf("Invalid digest at line %d\n", lineno)
		}
		return nil
	}
	return &Checksums{
		file: file,
		checksums: []*Checksum{
			&Checksum{
				hash: hash,
				csum: sum,
			},
		},
	}
}

func inputFromCheck(f io.ReadCloser) <-chan *Checksums {
	var current string
	var lineno uint64
	var input *Checksums
	var err error

	if f == nil {
		f = os.Stdin
		if opts.input != "" {
			if f, err = os.Open(opts.check); err != nil {
				logger.Fatal(err)
			}
		}
	}

	checksums := []*Checksum{}
	scanner := bufio.NewScanner(f)
	scanner.Split(bufio.ScanLines)

	files := make(chan *Checksums)
	go func() {
		for {
			lineno++
			eof := !scanner.Scan()
			if !eof {
				if input = parseLine(scanner.Text(), lineno); input != nil && current == "" {
					current = input.file
				}
			}
			if input != nil && current != input.file || eof {
				if current != "" {
					if best := bestHashes(checksums); best != nil {
						files <- &Checksums{file: current, checksums: best}
					}
				}
				if eof {
					if err := scanner.Err(); err != nil {
						logger.Fatal(err)
					}
					break
				}
				checksums = []*Checksum{}
			}
			if input != nil {
				checksums = append(checksums, input.checksums[0])
			}
		}
		close(files)
		f.Close()
	}()
	return files
}
<commit_msg>Fix bug with -c option not recognizing ""<commit_after>package main

import (
	"bufio"
	"crypto"
	"encoding/hex"
	"golang.org/x/exp/slices"
	"io"
	"os"
	"strings"
)

// Choose the algorithm with the longest digest size
func bestHash(checksums []*Checksum, ignore crypto.Hash) *Checksum {
	max := 0
	var best *Checksum
	for _, checksum := range checksums {
		if ignore != checksum.hash && checksum.hash.Size() > max {
			max = checksum.hash.Size()
			best = checksum
		}
	}
	return best
}

// Choose the best algorithm, or 2 if one of them is insecure
func bestHashes(checksums []*Checksum) (best []*Checksum) {
	best1 := bestHash(checksums, 0)
	if best1 == nil {
		return nil
	}
	best = append(best, best1)
	// Return 2 algorithms if the "best" of them is insecure
	if slices.Contains(insecure, best1.hash) {
		best2 := bestHash(checksums, best1.hash)
		if best2 != nil {
			best = append(best, best2)
		}
	}
	return best
}

func isChosen(h crypto.Hash) bool {
	for i := range chosen {
		if chosen[i].hash == h {
			return true
		}
	}
	return false
}

func parseLine(line string, lineno uint64) *Checksums {
	var algorithm, file, digest string
	var hash crypto.Hash
	var match []string
	var sum []byte
	var err error
	var ok bool

	if !opts.bsd {
		match = regex.gnu.FindStringSubmatch(line)
		if match != nil {
			digest, file = strings.ToLower(match[1]), unescapeFilename(match[2])
			if len(chosen) == 1 {
				algorithm = algorithms[chosen[0].hash].name
			} else {
				algorithm = size2hash[len(digest)]
			}
		}
	}
	if !opts.gnu {
		match = regex.bsd.FindStringSubmatch(line)
		if match != nil {
			algorithm, file, digest = match[1], match[2], strings.ToLower(match[3])
		}
	}
	sum, err = hex.DecodeString(digest)
	if hash, ok = name2Hash[algorithm]; !ok || err != nil || len(chosen) > 0 && !isChosen(hash) {
		stats.invalid++
		if opts.warn {
			logger.Printf("Invalid digest at line %d\n", lineno)
		} else if opts.strict {
			logger.Fatalf("Invalid digest at line %d\n", lineno)
		}
		return nil
	}
	return &Checksums{
		file: file,
		checksums: []*Checksum{
			&Checksum{
				hash: hash,
				csum: sum,
			},
		},
	}
}

func inputFromCheck(f io.ReadCloser) <-chan *Checksums {
	var current string
	var lineno uint64
	var input *Checksums
	var err error

	if f == nil {
		f = os.Stdin
		if opts.check != "" {
			if f, err = os.Open(opts.check); err != nil {
				logger.Fatal(err)
			}
		}
	}

	checksums := []*Checksum{}
	scanner := bufio.NewScanner(f)
	scanner.Split(bufio.ScanLines)

	files := make(chan *Checksums)
	go func() {
		for {
			lineno++
			eof := !scanner.Scan()
			if !eof {
				if input = parseLine(scanner.Text(), lineno); input != nil && current == "" {
					current = input.file
				}
			}
			if input != nil && current != input.file || eof {
				if current != "" {
					if best := bestHashes(checksums); best != nil {
						files <- &Checksums{file: current, checksums: best}
					}
				}
				if eof {
					if err := scanner.Err(); err != nil {
						logger.Fatal(err)
					}
					break
				}
				checksums = []*Checksum{}
			}
			if input != nil {
				checksums = append(checksums, input.checksums[0])
			}
		}
		close(files)
		f.Close()
	}()
	return files
}

{'fraction_non_alphanumeric': 0.14377952755905513, 'fraction_numerical': 0.008503937007874015, 'mean_word_length': 3.0069400630914824, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5943552', 'n_tokens_mistral': 2704, 'n_tokens_neox': 2338, 'n_words': 623}