<commit_before>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// package protoscope ...
package protoscope

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"math"
	"regexp"
	"strconv"
	"strings"

	_ "embed"
)

// The contents of language.text.
//
//go:embed language.txt
var LanguageTxt string

// A Position describes a location in the input stream.
//
// The zero-value Position represents the first byte of an anonymous input file.
type Position struct {
	Offset int    // Byte offset.
	Line   int    // Line number (zero-indexed).
	Column int    // Column number (zero-indexed byte, not rune, count).
	File   string // Optional file name for pretty-printing.
}

// String converts a Position to a string.
func (p Position) String() string {
	file := p.File
	if file == "" {
		file = "<input>"
	}
	return fmt.Sprintf("%s:%d:%d", file, p.Line+1, p.Column+1)
}

// A tokenKind is a kind of token.
type tokenKind int

const (
	tokenBytes tokenKind = iota
	tokenLongForm
	tokenLeftCurly
	tokenRightCurly
	tokenGroupCurly
	tokenEOF
)

// A ParseError may be produced while executing a Protoscope file, wrapping
// another error along with a position.
//
// Errors produced by functions in this package my by type-asserted to
// ParseError to try and obtain the position at which the error occurred.
type ParseError struct {
	Pos Position
	Err error
}

// Error makes this type into an error type.
func (e *ParseError) Error() string {
	return fmt.Sprintf("%s: %s", e.Pos, e.Err)
}

// Unwrap extracts the inner wrapped error.
//
// See errors.Unwrap().
func (e *ParseError) Unwrap() error {
	return e.Err
}

// A token is a token in a Protoscope file.
type token struct {
	// Kind is the kind of the token.
	Kind tokenKind
	// Value, for a tokenBytes token, is the decoded value of the token in
	// bytes.
	Value []byte
	// WireType, for a tokenBytes token, is which wire type an InferredType
	// tag expression that preceded it should become.
	WireType int
	// InferredType indicates that this was a tag expression which wishes to infer
	// its type based on tokens that follow.
	InferredType bool
	// Pos is the position of the first byte of the token.
	Pos Position
	// Length, for a tokenLongForm token, is the number of bytes to use to
	// encode the length, not including the initial one.
	Length int
	// FieldNumber, if not -1, indicates that this was a tag token. This is used
	// for implementing group syntax.
	FieldNumber int64
}

var (
	// The relevant capture groups are:
	// 1: The actual value.
	// 2: The encoding format.
	// 3: The wire type, including the colon, if this is a tag.
	// 4: The wire type expression, which may be empty if it is inferred.
	regexpIntOrTag = regexp.MustCompile(`^-?([0-9]+|0x[0-9a-fA-F]+)(z|i32|i64)?(:(\w*))?$`)
	regexpDecFp    = regexp.MustCompile(`^(-?[0-9]+\.[0-9]+(?:[eE]-?[0-9]+)?)(i32|i64)?$`)
	regexpHexFp    = regexp.MustCompile(`^(-?0x[0-9a-fA-F]+\.[0-9a-fA-F]+(?:[pP]-?[0-9]+)?)(i32|i64)?$`)
	regexpLongForm = regexp.MustCompile(`^long-form:([0-9]+)$`)
)

// A Scanner represents parsing state for a Protoscope file.
//
// A zero-value Scanner is ready to begin parsing (given that Input is set to
// a valid value). However, it is recommended to use NewScanner to create a new
// Scanner, since it can pre-populate fields other than Input with default
// settings.
type Scanner struct {
	// Input is the input text being processed.
	Input string
	// Position is the current position at which parsing should
	// resume. The Offset field is used for indexing into Input; the remaining
	// fields are used for error-reporting.
	pos Position
}

// NewScanner creates a new scanner for parsing the given input.
func NewScanner(input string) *Scanner {
	return &Scanner{Input: input}
}

// SetFile sets the file path shown in this Scanner's error reports.
func (s *Scanner) SetFile(path string) {
	s.pos.File = path
}

// Exec consumes tokens until Input is exhausted, returning the resulting
// encoded maybe-DER.
func (s *Scanner) Exec() ([]byte, error) {
	return s.exec(nil)
}

// isEOF returns whether the cursor is at least n bytes ahead of the end of the
// input.
func (s *Scanner) isEOF(n int) bool {
	return s.pos.Offset+n >= len(s.Input)
}

// advance advances the scanner's cursor n positions.
//
// Unlike just s.pos.Offset += n, this will not proceed beyond the end of the
// string, and will update the line and column information accordingly.
func (s *Scanner) advance(n int) {
	for i := 0; i < n && !s.isEOF(0); i++ {
		if s.Input[s.pos.Offset] == '\n' {
			s.pos.Line++
			s.pos.Column = 0
		} else {
			s.pos.Column++
		}
		s.pos.Offset++
	}
}

// consume advances exactly n times and returns all source bytes between the
// initial cursor position and excluding the final cursor position.
//
// If EOF is reached before all n bytes are consumed, the function returns
// false.
func (s *Scanner) consume(n int) (string, bool) {
	start := s.pos.Offset
	s.advance(n)
	if s.pos.Offset-start != n {
		return "", false
	}

	return s.Input[start:s.pos.Offset], true
}

// consumeUntil advances the cursor until the given byte is seen, returning all
// source bytes between the initial cursor position and excluding the given
// byte. This function will advance past the searched-for byte.
//
// If EOF is reached before the byte is seen, the function returns false.
func (s *Scanner) consumeUntil(b byte) (string, bool) {
	if i := strings.IndexByte(s.Input[s.pos.Offset:], b); i != -1 {
		text, _ := s.consume(i + 1)
		return text[:i], true
	}
	return "", false
}

// parseEscapeSequence parses a Protoscope escape sequence, returning the byte
// it escapes.
//
// Valid escapes are:
// \n \" \\ \xNN
//
// This function assumes that the scanner's cursor is currently on a \ rune.
func (s *Scanner) parseEscapeSequence() (byte, error) {
	s.advance(1) // Skip the \. The caller is assumed to have validated it.
	if s.isEOF(0) {
		return 0, &ParseError{s.pos, errors.New("expected escape character")}
	}

	switch c := s.Input[s.pos.Offset]; c {
	case 'n':
		s.advance(1)
		return '\n', nil
	case '"', '\\':
		s.advance(1)
		return c, nil
	case 'x':
		s.advance(1)

		hexes, ok := s.consume(2)
		if !ok {
			return 0, &ParseError{s.pos, errors.New("unfinished escape sequence")}
		}

		bytes, err := hex.DecodeString(hexes)
		if err != nil {
			return 0, &ParseError{s.pos, err}
		}

		var r byte
		for _, b := range bytes {
			r <<= 8
			r |= b
		}
		return r, nil
	case '0', '1', '2', '3', '4', '5', '6', '7':
		i := 0
		for i < 3 && !s.isEOF(0) {
			c := s.Input[s.pos.Offset+i]
			if c < '0' || c > '7' {
				break
			}
			i++
		}
		str := s.Input[s.pos.Offset : s.pos.Offset+i]
		r, err := strconv.ParseUint(str, 8, 8)
		s.advance(i)
		return byte(r), err
	default:
		return 0, &ParseError{s.pos, fmt.Errorf("unknown escape sequence \\%c", c)}
	}
}

// parseQuotedString parses a UTF-8 string until the next ".
//
// This function assumes that the scanner's cursor is currently on a " rune.
func (s *Scanner) parseQuotedString() (token, error) {
	s.advance(1) // Skip the ". The caller is assumed to have validated it.
	start := s.pos
	var bytes []byte
	for {
		if s.isEOF(0) {
			return token{}, &ParseError{start, errors.New("unmatched \"")}
		}
		switch c := s.Input[s.pos.Offset]; c {
		case '"':<commit_msg>add support for octal escape sequences<commit_after>// Copyright 2022 Google LLC
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//      http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

// package protoscope ...
package protoscope

import (
	"encoding/binary"
	"encoding/hex"
	"errors"
	"fmt"
	"math"
	"regexp"
	"strconv"
	"strings"

	_ "embed"
)

// The contents of language.text.
//
//go:embed language.txt
var LanguageTxt string

// A Position describes a location in the input stream.
//
// The zero-value Position represents the first byte of an anonymous input file.
type Position struct {
	Offset int    // Byte offset.
	Line   int    // Line number (zero-indexed).
	Column int    // Column number (zero-indexed byte, not rune, count).
	File   string // Optional file name for pretty-printing.
}

// String converts a Position to a string.
func (p Position) String() string {
	file := p.File
	if file == "" {
		file = "<input>"
	}
	return fmt.Sprintf("%s:%d:%d", file, p.Line+1, p.Column+1)
}

// A tokenKind is a kind of token.
type tokenKind int

const (
	tokenBytes tokenKind = iota
	tokenLongForm
	tokenLeftCurly
	tokenRightCurly
	tokenGroupCurly
	tokenEOF
)

// A ParseError may be produced while executing a Protoscope file, wrapping
// another error along with a position.
//
// Errors produced by functions in this package my by type-asserted to
// ParseError to try and obtain the position at which the error occurred.
type ParseError struct {
	Pos Position
	Err error
}

// Error makes this type into an error type.
func (e *ParseError) Error() string {
	return fmt.Sprintf("%s: %s", e.Pos, e.Err)
}

// Unwrap extracts the inner wrapped error.
//
// See errors.Unwrap().
func (e *ParseError) Unwrap() error {
	return e.Err
}

// A token is a token in a Protoscope file.
type token struct {
	// Kind is the kind of the token.
	Kind tokenKind
	// Value, for a tokenBytes token, is the decoded value of the token in
	// bytes.
	Value []byte
	// WireType, for a tokenBytes token, is which wire type an InferredType
	// tag expression that preceded it should become.
	WireType int
	// InferredType indicates that this was a tag expression which wishes to infer
	// its type based on tokens that follow.
	InferredType bool
	// Pos is the position of the first byte of the token.
	Pos Position
	// Length, for a tokenLongForm token, is the number of bytes to use to
	// encode the length, not including the initial one.
	Length int
	// FieldNumber, if not -1, indicates that this was a tag token. This is used
	// for implementing group syntax.
	FieldNumber int64
}

var (
	// The relevant capture groups are:
	// 1: The actual value.
	// 2: The encoding format.
	// 3: The wire type, including the colon, if this is a tag.
	// 4: The wire type expression, which may be empty if it is inferred.
	regexpIntOrTag = regexp.MustCompile(`^-?([0-9]+|0x[0-9a-fA-F]+)(z|i32|i64)?(:(\w*))?$`)
	regexpDecFp    = regexp.MustCompile(`^(-?[0-9]+\.[0-9]+(?:[eE]-?[0-9]+)?)(i32|i64)?$`)
	regexpHexFp    = regexp.MustCompile(`^(-?0x[0-9a-fA-F]+\.[0-9a-fA-F]+(?:[pP]-?[0-9]+)?)(i32|i64)?$`)
	regexpLongForm = regexp.MustCompile(`^long-form:([0-9]+)$`)
)

// A Scanner represents parsing state for a Protoscope file.
//
// A zero-value Scanner is ready to begin parsing (given that Input is set to
// a valid value). However, it is recommended to use NewScanner to create a new
// Scanner, since it can pre-populate fields other than Input with default
// settings.
type Scanner struct {
	// Input is the input text being processed.
	Input string
	// Position is the current position at which parsing should
	// resume. The Offset field is used for indexing into Input; the remaining
	// fields are used for error-reporting.
	pos Position
}

// NewScanner creates a new scanner for parsing the given input.
func NewScanner(input string) *Scanner {
	return &Scanner{Input: input}
}

// SetFile sets the file path shown in this Scanner's error reports.
func (s *Scanner) SetFile(path string) {
	s.pos.File = path
}

// Exec consumes tokens until Input is exhausted, returning the resulting
// encoded maybe-DER.
func (s *Scanner) Exec() ([]byte, error) {
	return s.exec(nil)
}

// isEOF returns whether the cursor is at least n bytes ahead of the end of the
// input.
func (s *Scanner) isEOF(n int) bool {
	return s.pos.Offset+n >= len(s.Input)
}

// advance advances the scanner's cursor n positions.
//
// Unlike just s.pos.Offset += n, this will not proceed beyond the end of the
// string, and will update the line and column information accordingly.
func (s *Scanner) advance(n int) {
	for i := 0; i < n && !s.isEOF(0); i++ {
		if s.Input[s.pos.Offset] == '\n' {
			s.pos.Line++
			s.pos.Column = 0
		} else {
			s.pos.Column++
		}
		s.pos.Offset++
	}
}

// consume advances exactly n times and returns all source bytes between the
// initial cursor position and excluding the final cursor position.
//
// If EOF is reached before all n bytes are consumed, the function returns
// false.
func (s *Scanner) consume(n int) (string, bool) {
	start := s.pos.Offset
	s.advance(n)
	if s.pos.Offset-start != n {
		return "", false
	}

	return s.Input[start:s.pos.Offset], true
}

// consumeUntil advances the cursor until the given byte is seen, returning all
// source bytes between the initial cursor position and excluding the given
// byte. This function will advance past the searched-for byte.
//
// If EOF is reached before the byte is seen, the function returns false.
func (s *Scanner) consumeUntil(b byte) (string, bool) {
	if i := strings.IndexByte(s.Input[s.pos.Offset:], b); i != -1 {
		text, _ := s.consume(i + 1)
		return text[:i], true
	}
	return "", false
}

// parseEscapeSequence parses a Protoscope escape sequence, returning the byte
// it escapes.
//
// Valid escapes are:
// \n \" \\ \xNN \NNN
//
// This function assumes that the scanner's cursor is currently on a \ rune.
func (s *Scanner) parseEscapeSequence() (byte, error) {
	s.advance(1) // Skip the \. The caller is assumed to have validated it.
	if s.isEOF(0) {
		return 0, &ParseError{s.pos, errors.New("expected escape character")}
	}

	switch c := s.Input[s.pos.Offset]; c {
	case 'n':
		s.advance(1)
		return '\n', nil
	case '"', '\\':
		s.advance(1)
		return c, nil
	case 'x':
		s.advance(1)

		hexes, ok := s.consume(2)
		if !ok {
			return 0, &ParseError{s.pos, errors.New("unfinished escape sequence")}
		}

		bytes, err := hex.DecodeString(hexes)
		if err != nil {
			return 0, &ParseError{s.pos, err}
		}

		var r byte
		for _, b := range bytes {
			r <<= 8
			r |= b
		}
		return r, nil
	case '0', '1', '2', '3', '4', '5', '6', '7':
		start := s.pos.Offset
		for i := 0; i < 3 && !s.isEOF(0); i++ {
			c := s.Input[s.pos.Offset]
			if c < '0' || c > '7' {
				break
			}
			s.advance(1)
		}
		str := s.Input[start:s.pos.Offset]
		r, err := strconv.ParseUint(str, 8, 8)
		if err != nil {
			return 0, &ParseError{s.pos, err}
		}
		return byte(r), nil
	default:
		return 0, &ParseError{s.pos, fmt.Errorf("unknown escape sequence \\%c", c)}
	}
}

// parseQuotedString parses a UTF-8 string until the next ".
//
// This function assumes that the scanner's cursor is currently on a " rune.
func (s *Scanner) parseQuotedString() (token, error) {
	s.advance(1) // Skip the ". The caller is assumed to have validated it.
	start := s.pos
	var bytes []byte
	for {
		if s.isEOF(0) {
			return token{}, &ParseError{start, errors.New("unmatched \"")}
		}
		switch c := s.Input[s.pos.Offset]; c {
		case '"':
{'fraction_non_alphanumeric': 0.12999619337647506, 'fraction_numerical': 0.010658545869813475, 'mean_word_length': 3.1867197875166005, 'pattern_counts': {'":': 0, '<': 15, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5575975', 'n_tokens_mistral': 5435, 'n_tokens_neox': 5158, 'n_words': 2020}