<commit_before>package lfs

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io"

	"github.com/git-lfs/git-lfs/git"
)

// runCatFileBatch uses 'git cat-file --batch' to get the object contents of a
// git object, given its sha1. The contents will be decoded into a Git LFS
// pointer. Git Blob SHA1s are read from the sha1Ch channel and fed to STDIN.
// Results are parsed from STDOUT, and any eligible LFS pointers are sent to
// pointerCh. If a Git Blob is not an LFS pointer, check the lockableSet to see
// if that blob is for a locked file. Any errors are sent to errCh. An error is
// returned if the 'git cat-file' command fails to start.
func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error) error {
	scanner, err := NewCatFileBatchScanner()
	if err != nil {
		return err
	}

	go func() {
		for r := range revs.Results {
			canScan := scanner.Scan(r)

			if err := scanner.Err(); err != nil {
				errCh <- err
			} else if p := scanner.Pointer(); p != nil {
				pointerCh <- p
			} else if b := scanner.BlobSHA(); len(b) == 40 {
				if name, ok := lockableSet.Check(b); ok {
					lockableCh <- name
				}
			}

			if !canScan {
				break
			}
		}

		if err := revs.Wait(); err != nil {
			errCh <- err
		}

		if err := scanner.Close(); err != nil {
			errCh <- err
		}

		close(pointerCh)
		close(errCh)
		close(lockableCh)
	}()

	return nil
}

type CatFileBatchScanner struct {
	scanner *git.ObjectScanner

	blobSha     string
	contentsSha string
	pointer     *WrappedPointer
	err         error
}

func NewCatFileBatchScanner() (*CatFileBatchScanner, error) {
	scanner, err := git.NewObjectScanner()
	if err != nil {
		return nil, err
	}

	return &CatFileBatchScanner{scanner: scanner}, nil
}

func (s *CatFileBatchScanner) BlobSHA() string {
	return s.blobSha
}

func (s *CatFileBatchScanner) ContentsSha() string {
	return s.contentsSha
}

func (s *CatFileBatchScanner) Pointer() *WrappedPointer {
	return s.pointer
}

func (s *CatFileBatchScanner) Err() error {
	return s.err
}

func (s *CatFileBatchScanner) Scan(sha string) bool {
	s.pointer, s.err = nil, nil
	s.blobSha, s.contentsSha = "", ""

	b, c, p, err := s.next(sha)
	s.blobSha = b
	s.contentsSha = c
	s.pointer = p

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}

	fmt.Println("returning true with blobsha:", b, "contents:", c, err)
	return true
}

func (s *CatFileBatchScanner) Close() error {
	return s.scanner.Close()
}

func (s *CatFileBatchScanner) next(blob string) (string, string, *WrappedPointer, error) {
	if !s.scanner.Scan(blob) {<commit_msg>lfs: remove some unneeded debugging information<commit_after>package lfs

import (
	"bytes"
	"crypto/sha256"
	"fmt"
	"io"

	"github.com/git-lfs/git-lfs/git"
)

// runCatFileBatch uses 'git cat-file --batch' to get the object contents of a
// git object, given its sha1. The contents will be decoded into a Git LFS
// pointer. Git Blob SHA1s are read from the sha1Ch channel and fed to STDIN.
// Results are parsed from STDOUT, and any eligible LFS pointers are sent to
// pointerCh. If a Git Blob is not an LFS pointer, check the lockableSet to see
// if that blob is for a locked file. Any errors are sent to errCh. An error is
// returned if the 'git cat-file' command fails to start.
func runCatFileBatch(pointerCh chan *WrappedPointer, lockableCh chan string, lockableSet *lockableNameSet, revs *StringChannelWrapper, errCh chan error) error {
	scanner, err := NewCatFileBatchScanner()
	if err != nil {
		return err
	}

	go func() {
		for r := range revs.Results {
			canScan := scanner.Scan(r)

			if err := scanner.Err(); err != nil {
				errCh <- err
			} else if p := scanner.Pointer(); p != nil {
				pointerCh <- p
			} else if b := scanner.BlobSHA(); len(b) == 40 {
				if name, ok := lockableSet.Check(b); ok {
					lockableCh <- name
				}
			}

			if !canScan {
				break
			}
		}

		if err := revs.Wait(); err != nil {
			errCh <- err
		}

		if err := scanner.Close(); err != nil {
			errCh <- err
		}

		close(pointerCh)
		close(errCh)
		close(lockableCh)
	}()

	return nil
}

type CatFileBatchScanner struct {
	scanner *git.ObjectScanner

	blobSha     string
	contentsSha string
	pointer     *WrappedPointer
	err         error
}

func NewCatFileBatchScanner() (*CatFileBatchScanner, error) {
	scanner, err := git.NewObjectScanner()
	if err != nil {
		return nil, err
	}

	return &CatFileBatchScanner{scanner: scanner}, nil
}

func (s *CatFileBatchScanner) BlobSHA() string {
	return s.blobSha
}

func (s *CatFileBatchScanner) ContentsSha() string {
	return s.contentsSha
}

func (s *CatFileBatchScanner) Pointer() *WrappedPointer {
	return s.pointer
}

func (s *CatFileBatchScanner) Err() error {
	return s.err
}

func (s *CatFileBatchScanner) Scan(sha string) bool {
	s.pointer, s.err = nil, nil
	s.blobSha, s.contentsSha = "", ""

	b, c, p, err := s.next(sha)
	s.blobSha = b
	s.contentsSha = c
	s.pointer = p

	if err != nil {
		if err != io.EOF {
			s.err = err
		}
		return false
	}

	return true
}

func (s *CatFileBatchScanner) Close() error {
	return s.scanner.Close()
}

func (s *CatFileBatchScanner) next(blob string) (string, string, *WrappedPointer, error) {
	if !s.scanner.Scan(blob) {
{'fraction_non_alphanumeric': 0.11932357970739123, 'fraction_numerical': 0.0030400912027360822, 'mean_word_length': 3.3006535947712417, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7366441', 'n_tokens_mistral': 2036, 'n_tokens_neox': 1857, 'n_words': 601}