<commit_before>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syntax

import (
	"cmd/internal/src"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const debug = false
const trace = false

type parser struct {
	base  *src.PosBase
	errh  ErrorHandler
	fileh FilenameHandler
	mode  Mode
	scanner

	first  error  // first error encountered
	errcnt int    // number of errors encountered
	pragma Pragma // pragma flags

	fnest  int    // function nesting level (for error handling)
	xnest  int    // expression nesting level (for complit ambiguity resolution)
	indent []byte // tracing support
}

func (p *parser) init(base *src.PosBase, r io.Reader, errh ErrorHandler, pragh PragmaHandler, fileh FilenameHandler, mode Mode) {
	p.base = base
	p.errh = errh
	p.fileh = fileh
	p.mode = mode
	p.scanner.init(
		r,
		// Error and directive handler for scanner.
		// Because the (line, col) positions passed to the
		// handler is always at or after the current reading
		// position, it is safe to use the most recent position
		// base to compute the corresponding Pos value.
		func(line, col uint, msg string) {
			if msg[0] != '/' {
				p.errorAt(p.posAt(line, col), msg)
				return
			}

			// otherwise it must be a comment containing a line or go: directive
			text := commentText(msg)
			col += 2 // text starts after // or /*
			if strings.HasPrefix(text, "line ") {
				p.updateBase(line, col+5, text[5:])
				return
			}

			// go: directive (but be conservative and test)
			if pragh != nil && strings.HasPrefix(text, "go:") {
				p.pragma |= pragh(p.posAt(line, col), text)
			}
		},
		directives,
	)

	p.first = nil
	p.errcnt = 0
	p.pragma = 0

	p.fnest = 0
	p.xnest = 0
	p.indent = nil
}

const lineMax = 1<<24 - 1 // TODO(gri) this limit is defined for src.Pos - fix

func (p *parser) updateBase(line, col uint, text string) {
	i, n, ok := trailingDigits(text)
	if i == 0 {
		return // ignore (not a line directive)
	}
	// i > 0

	if !ok {
		// text has a suffix :xxx but xxx is not a number
		p.errorAt(p.posAt(line, col+i), "invalid line number: "+text[i:])
		return
	}

	i2, n2, ok2 := trailingDigits(text[:i-1])
	if ok2 {
		//line filename:line:col
		i, i2 = i2, i
		n, n2 = n2, n
		if n2 == 0 {
			p.errorAt(p.posAt(line, col+i2), "invalid column number: "+text[i2:])
			return
		}
		text = text[:i2-1] // lop off :col
	}

	if n == 0 || n > lineMax {
		p.errorAt(p.posAt(line, col+i), "invalid line number: "+text[i:])
		return
	}

	filename := text[:i-1] // lop off :line
	absFilename := filename
	if p.fileh != nil {
		absFilename = p.fileh(filename)
	}

	// TODO(gri) pass column n2 to NewLinePragmaBase
	p.base = src.NewLinePragmaBase(src.MakePos(p.base.Pos().Base(), line, col), filename, absFilename, uint(n) /*uint(n2)*/)
}

func commentText(s string) string {
	if s[:2] == "/*" {
		return s[2 : len(s)-2] // lop off /* and */
	}

	// line comment (does not include newline)
	// (on Windows, the line comment may end in \r\n)
	i := len(s)
	if s[i-1] == '\r' {
		i--
	}
	return s[2:i] // lop off // and \r at end, if any
}

func trailingDigits(text string) (uint, uint, bool) {
	// Want to use LastIndexByte below but it's not defined in Go1.4 and bootstrap fails.
	i := strings.LastIndex(text, ":") // look from right (Windows filenames may contain ':')
	if i < 0 {<commit_msg>cmd/compile/internal/syntax: simpler position base update for line directives (cleanup)<commit_after>// Copyright 2016 The Go Authors. All rights reserved.
// Use of this source code is governed by a BSD-style
// license that can be found in the LICENSE file.

package syntax

import (
	"cmd/internal/src"
	"fmt"
	"io"
	"strconv"
	"strings"
)

const debug = false
const trace = false

type parser struct {
	file  *src.PosBase
	errh  ErrorHandler
	fileh FilenameHandler
	mode  Mode
	scanner

	base   *src.PosBase // current position base
	first  error        // first error encountered
	errcnt int          // number of errors encountered
	pragma Pragma       // pragma flags

	fnest  int    // function nesting level (for error handling)
	xnest  int    // expression nesting level (for complit ambiguity resolution)
	indent []byte // tracing support
}

func (p *parser) init(file *src.PosBase, r io.Reader, errh ErrorHandler, pragh PragmaHandler, fileh FilenameHandler, mode Mode) {
	p.file = file
	p.errh = errh
	p.fileh = fileh
	p.mode = mode
	p.scanner.init(
		r,
		// Error and directive handler for scanner.
		// Because the (line, col) positions passed to the
		// handler is always at or after the current reading
		// position, it is safe to use the most recent position
		// base to compute the corresponding Pos value.
		func(line, col uint, msg string) {
			if msg[0] != '/' {
				p.errorAt(p.posAt(line, col), msg)
				return
			}

			// otherwise it must be a comment containing a line or go: directive
			text := commentText(msg)
			col += 2 // text starts after // or /*
			if strings.HasPrefix(text, "line ") {
				p.updateBase(line, col+5, text[5:])
				return
			}

			// go: directive (but be conservative and test)
			if pragh != nil && strings.HasPrefix(text, "go:") {
				p.pragma |= pragh(p.posAt(line, col), text)
			}
		},
		directives,
	)

	p.base = file
	p.first = nil
	p.errcnt = 0
	p.pragma = 0

	p.fnest = 0
	p.xnest = 0
	p.indent = nil
}

const lineMax = 1<<24 - 1 // TODO(gri) this limit is defined for src.Pos - fix

func (p *parser) updateBase(line, col uint, text string) {
	i, n, ok := trailingDigits(text)
	if i == 0 {
		return // ignore (not a line directive)
	}
	// i > 0

	if !ok {
		// text has a suffix :xxx but xxx is not a number
		p.errorAt(p.posAt(line, col+i), "invalid line number: "+text[i:])
		return
	}

	i2, n2, ok2 := trailingDigits(text[:i-1])
	if ok2 {
		//line filename:line:col
		i, i2 = i2, i
		n, n2 = n2, n
		if n2 == 0 {
			p.errorAt(p.posAt(line, col+i2), "invalid column number: "+text[i2:])
			return
		}
		text = text[:i2-1] // lop off :col
	}

	if n == 0 || n > lineMax {
		p.errorAt(p.posAt(line, col+i), "invalid line number: "+text[i:])
		return
	}

	filename := text[:i-1] // lop off :line
	absFilename := filename
	if p.fileh != nil {
		absFilename = p.fileh(filename)
	}

	// TODO(gri) pass column n2 to NewLinePragmaBase
	p.base = src.NewLinePragmaBase(src.MakePos(p.file, line, col), filename, absFilename, uint(n) /*uint(n2)*/)
}

func commentText(s string) string {
	if s[:2] == "/*" {
		return s[2 : len(s)-2] // lop off /* and */
	}

	// line comment (does not include newline)
	// (on Windows, the line comment may end in \r\n)
	i := len(s)
	if s[i-1] == '\r' {
		i--
	}
	return s[2:i] // lop off // and \r at end, if any
}

func trailingDigits(text string) (uint, uint, bool) {
	// Want to use LastIndexByte below but it's not defined in Go1.4 and bootstrap fails.
	i := strings.LastIndex(text, ":") // look from right (Windows filenames may contain ':')
	if i < 0 {
{'fraction_non_alphanumeric': 0.1355005753739931, 'fraction_numerical': 0.012945914844649022, 'mean_word_length': 2.895238095238095, 'pattern_counts': {'":': 2, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1678035', 'n_tokens_mistral': 2703, 'n_tokens_neox': 2483, 'n_words': 857}