<commit_before>package dependency

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
	"jrubin.io/slog"
	"jrubin.io/zb/lib/zbcontext"
)

var _ Dependency = (*GoFile)(nil)

type GoFile struct {
	*zbcontext.Context
	Path              string
	ProjectImportPath string

	mu           sync.RWMutex
	dependencies []Dependency
}

var goFileCache = map[string]*GoFile{}
var goFileCacheMu sync.RWMutex

func NewGoFile(ctx *zbcontext.Context, projectimportPath, path string) *GoFile {
	goFileCacheMu.RLock()

	if f, ok := goFileCache[path]; ok {
		goFileCacheMu.RUnlock()
		return f
	}

	goFileCacheMu.RUnlock()

	goFileCacheMu.Lock()
	defer goFileCacheMu.Unlock()

	if f, ok := goFileCache[path]; ok {
		return f
	}

	f := &GoFile{
		Context:           ctx,
		Path:              path,
		ProjectImportPath: projectimportPath,
	}

	goFileCache[path] = f

	return f
}

func (e *GoFile) Name() string {
	return e.Path
}

func (e *GoFile) ModTime() time.Time {
	i, err := os.Stat(e.Path)
	if err != nil {
		return time.Time{}
	}
	return i.ModTime()
}

func isZBGenerate(buf []byte) bool {
	return bytes.HasPrefix(buf, []byte("//zb:generate ")) ||
		bytes.HasPrefix(buf, []byte("//zb:generate\t"))
}

func isTodoOrFixme(buf []byte) bool {
	return bytes.Contains(buf, []byte(strings.ToUpper("todo"))) ||
		bytes.Contains(buf, []byte(strings.ToUpper("fixme")))
}

func (e *GoFile) Dependencies() ([]Dependency, error) {
	e.mu.RLock()

	if e.dependencies != nil {
		defer e.mu.RUnlock()
		return e.dependencies, nil
	}

	e.mu.RUnlock()

	e.mu.Lock()
	defer e.mu.Unlock()

	if e.dependencies != nil {
		return e.dependencies, nil
	}

	e.dependencies = []Dependency{}

	file, err := os.Open(e.Path)
	if err != nil {
		return nil, err
	}
	defer func() { _ = file.Close() }() // nosec

	// the following loop taken largely from go source src/cmd/go/generate.go

	// Scan for lines that start "//zb:generate".
	// Can't use bufio.Scanner because it can't handle long lines,
	// which are likely to appear when using generate.
	input := bufio.NewReader(file)
	// One line per loop.
	for i := 1; ; i++ {
		var buf []byte
		buf, err = input.ReadSlice('\n')
		if err == bufio.ErrBufferFull {
			// Line too long - consume and ignore.
			if isZBGenerate(buf) {
				return nil, errors.Errorf("zb:generate directive too long")
			}
			for err == bufio.ErrBufferFull {
				_, err = input.ReadSlice('\n')
			}
			if err != nil {
				return nil, err
			}
			continue
		}

		if err != nil {
			// Check for marker at EOF without final \n.
			if err == io.EOF && isZBGenerate(buf) {
				err = io.ErrUnexpectedEOF
			}
			break
		}

		if !e.NoWarnTodoFixme {
			base := e.Context.ImportPathToDir(e.ProjectImportPath) + string(filepath.Separator)
			if strings.HasPrefix(e.Path, base) &&
				!strings.Contains(e.Path, "vendor/") &&
				isTodoOrFixme(buf) {
				e.Logger.Warn(fmt.Sprintf("%s:%d:%s",
					strings.TrimPrefix(e.Path, base),
					i,
					strings.TrimSpace(string(buf)),
				))
			}
		}

		if !isZBGenerate(buf) {
			continue
		}

		var words []string
		words, err = split(string(buf))
		if err != nil {
			return nil, err
		}

		if len(words) == 0 {
			return nil, errors.New("no arguments to directive")
		}

		var deps []*GoGenerateFile
		deps, err = e.parseZBGenerate(words)
		if err != nil {
			return nil, err
		}

		prefix := filepath.Dir(e.Path) + string(filepath.Separator)
		for _, dep := range deps {
			e.Logger.WithFields(slog.Fields{
				"source":       strings.TrimPrefix(dep.Path, prefix),
				"depends_on":   strings.TrimPrefix(dep.Depends.Name(), prefix),
				"from_go_file": strings.TrimPrefix(e.Path, prefix),
			}).Debug("found go:generate dependency")

			e.dependencies = append(e.dependencies, dep)
		}
	}
	if err != nil && err != io.EOF {
		return nil, errors.Wrapf(err, "error reading %s", e.Path)
	}

	return e.dependencies, nil
}

func split(line string) ([]string, error) {
	// Parse line, obeying quoted strings.
	var words []string
	line = line[len("//zb:generate ") : len(line)-1] // Drop preamble and final newline.
	// There may still be a carriage return.
	if len(line) > 0 && line[len(line)-1] == '\r' {
		line = line[:len(line)-1]
	}
	// One (possibly quoted) word per iteration.
Words:
	for {
		line = strings.TrimLeft(line, " \t")
		if len(line) == 0 {
			break
		}
		if line[0] == '"' {
			for i := 1; i < len(line); i++ {
				c := line[i] // Only looking for ASCII so this is OK.<commit_msg>update go:generate log message<commit_after>package dependency

import (
	"bufio"
	"bytes"
	"fmt"
	"io"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"

	"github.com/pkg/errors"
	"jrubin.io/slog"
	"jrubin.io/zb/lib/zbcontext"
)

var _ Dependency = (*GoFile)(nil)

type GoFile struct {
	*zbcontext.Context
	Path              string
	ProjectImportPath string

	mu           sync.RWMutex
	dependencies []Dependency
}

var goFileCache = map[string]*GoFile{}
var goFileCacheMu sync.RWMutex

func NewGoFile(ctx *zbcontext.Context, projectimportPath, path string) *GoFile {
	goFileCacheMu.RLock()

	if f, ok := goFileCache[path]; ok {
		goFileCacheMu.RUnlock()
		return f
	}

	goFileCacheMu.RUnlock()

	goFileCacheMu.Lock()
	defer goFileCacheMu.Unlock()

	if f, ok := goFileCache[path]; ok {
		return f
	}

	f := &GoFile{
		Context:           ctx,
		Path:              path,
		ProjectImportPath: projectimportPath,
	}

	goFileCache[path] = f

	return f
}

func (e *GoFile) Name() string {
	return e.Path
}

func (e *GoFile) ModTime() time.Time {
	i, err := os.Stat(e.Path)
	if err != nil {
		return time.Time{}
	}
	return i.ModTime()
}

func isZBGenerate(buf []byte) bool {
	return bytes.HasPrefix(buf, []byte("//zb:generate ")) ||
		bytes.HasPrefix(buf, []byte("//zb:generate\t"))
}

func isTodoOrFixme(buf []byte) bool {
	return bytes.Contains(buf, []byte(strings.ToUpper("todo"))) ||
		bytes.Contains(buf, []byte(strings.ToUpper("fixme")))
}

func (e *GoFile) Dependencies() ([]Dependency, error) {
	e.mu.RLock()

	if e.dependencies != nil {
		defer e.mu.RUnlock()
		return e.dependencies, nil
	}

	e.mu.RUnlock()

	e.mu.Lock()
	defer e.mu.Unlock()

	if e.dependencies != nil {
		return e.dependencies, nil
	}

	e.dependencies = []Dependency{}

	file, err := os.Open(e.Path)
	if err != nil {
		return nil, err
	}
	defer func() { _ = file.Close() }() // nosec

	// the following loop taken largely from go source src/cmd/go/generate.go

	// Scan for lines that start "//zb:generate".
	// Can't use bufio.Scanner because it can't handle long lines,
	// which are likely to appear when using generate.
	input := bufio.NewReader(file)
	// One line per loop.
	for i := 1; ; i++ {
		var buf []byte
		buf, err = input.ReadSlice('\n')
		if err == bufio.ErrBufferFull {
			// Line too long - consume and ignore.
			if isZBGenerate(buf) {
				return nil, errors.Errorf("zb:generate directive too long")
			}
			for err == bufio.ErrBufferFull {
				_, err = input.ReadSlice('\n')
			}
			if err != nil {
				return nil, err
			}
			continue
		}

		if err != nil {
			// Check for marker at EOF without final \n.
			if err == io.EOF && isZBGenerate(buf) {
				err = io.ErrUnexpectedEOF
			}
			break
		}

		if !e.NoWarnTodoFixme {
			base := e.Context.ImportPathToDir(e.ProjectImportPath) + string(filepath.Separator)
			if strings.HasPrefix(e.Path, base) &&
				!strings.Contains(e.Path, "vendor/") &&
				isTodoOrFixme(buf) {
				e.Logger.Warn(fmt.Sprintf("%s:%d:%s",
					strings.TrimPrefix(e.Path, base),
					i,
					strings.TrimSpace(string(buf)),
				))
			}
		}

		if !isZBGenerate(buf) {
			continue
		}

		var words []string
		words, err = split(string(buf))
		if err != nil {
			return nil, err
		}

		if len(words) == 0 {
			return nil, errors.New("no arguments to directive")
		}

		var deps []*GoGenerateFile
		deps, err = e.parseZBGenerate(words)
		if err != nil {
			return nil, err
		}

		for _, dep := range deps {
			var source string
			source, err = filepath.Rel(e.SrcDir, dep.Path)
			if err != nil {
				source = dep.Path
			}

			var dependsOn string
			dependsOn, err = filepath.Rel(e.SrcDir, dep.Depends.Name())
			if err != nil {
				dependsOn = dep.Depends.Name()
			}

			var fromGo string
			fromGo, err = filepath.Rel(e.SrcDir, e.Path)
			if err != nil {
				fromGo = e.Path
			}

			e.Logger.WithFields(slog.Fields{
				"source":       source,
				"depends_on":   dependsOn,
				"from_go_file": fromGo,
			}).Debug("found go:generate dependency")

			e.dependencies = append(e.dependencies, dep)
		}
	}
	if err != nil && err != io.EOF {
		return nil, errors.Wrapf(err, "error reading %s", e.Path)
	}

	return e.dependencies, nil
}

func split(line string) ([]string, error) {
	// Parse line, obeying quoted strings.
	var words []string
	line = line[len("//zb:generate ") : len(line)-1] // Drop preamble and final newline.
	// There may still be a carriage return.
	if len(line) > 0 && line[len(line)-1] == '\r' {
		line = line[:len(line)-1]
	}
	// One (possibly quoted) word per iteration.
Words:
	for {
		line = strings.TrimLeft(line, " \t")
		if len(line) == 0 {
			break
		}
		if line[0] == '"' {
			for i := 1; i < len(line); i++ {
				c := line[i] // Only looking for ASCII so this is OK.
{'fraction_non_alphanumeric': 0.1535035306898425, 'fraction_numerical': 0.001955458989679522, 'mean_word_length': 3.037719298245614, 'pattern_counts': {'":': 6, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '84799', 'n_tokens_mistral': 3781, 'n_tokens_neox': 3376, 'n_words': 850}