<commit_before>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Build the parser:
//go:generate goyacc -v y.output -o parser.go -p mtail parser.y

package vm

import (
	"io"
	"path/filepath"
	"regexp"

	"github.com/google/mtail/metrics"
)

// Options contains all the parameters that affect the behaviour of the compiler.
type Options struct {
	CompileOnly          bool // Do not start the program after compilation.
	SyslogUseCurrentYear bool // Use the current year if no year is present in the log file timestamp.
	EmitAst              bool // Print the AST after parse
	EmitAstTypes         bool // Print the AST with types after typechecking
}

// Compile compiles a program from the input into a virtual machine or a list
// of compile errors.  It takes the program's name and the metric store as
// additional arguments to build the virtual machine.
func Compile(name string, input io.Reader, o *Options) (*VM, error) {
	name = filepath.Base(name)

	ast, err := Parse(name, input)
	if err != nil {
		return nil, err
	}
	if o.EmitAst {
		u := Unparser{}
		u.Unparse(ast)
	}

	if err := Check(ast); err != nil {
		return nil, err
	}
	if o.EmitAstTypes {
		u := Unparser{}
		u.emitTypes = true
		u.Unparse(ast)
	}

	obj, err := CodeGen(name, ast)
	if err != nil {
		return nil, err
	}

	if o.CompileOnly {
		return nil, nil
	}
<commit_msg>Dump the unparsed program and with types when asked.<commit_after>// Copyright 2011 Google Inc. All Rights Reserved.
// This file is available under the Apache license.

// Build the parser:
//go:generate goyacc -v y.output -o parser.go -p mtail parser.y

package vm

import (
	"io"
	"path/filepath"
	"regexp"

	"github.com/golang/glog"
	"github.com/google/mtail/metrics"
)

// Options contains all the parameters that affect the behaviour of the compiler.
type Options struct {
	CompileOnly          bool // Do not start the program after compilation.
	SyslogUseCurrentYear bool // Use the current year if no year is present in the log file timestamp.
	EmitAst              bool // Print the AST after parse
	EmitAstTypes         bool // Print the AST with types after typechecking
}

// Compile compiles a program from the input into a virtual machine or a list
// of compile errors.  It takes the program's name and the metric store as
// additional arguments to build the virtual machine.
func Compile(name string, input io.Reader, o *Options) (*VM, error) {
	name = filepath.Base(name)

	ast, err := Parse(name, input)
	if err != nil {
		return nil, err
	}
	if o.EmitAst {
		u := Unparser{}
		glog.Infof("%s AST:\n%s", name, u.Unparse(ast))
	}

	if err := Check(ast); err != nil {
		return nil, err
	}
	if o.EmitAstTypes {
		u := Unparser{}
		u.emitTypes = true
		glog.Infof("%s AST with Types:\n%s", name, u.Unparse(ast))
	}

	obj, err := CodeGen(name, ast)
	if err != nil {
		return nil, err
	}

	if o.CompileOnly {
		return nil, nil
	}

{'fraction_non_alphanumeric': 0.10047522063815342, 'fraction_numerical': 0.0027155465037338763, 'mean_word_length': 3.0425240054869684, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7609612', 'n_tokens_mistral': 985, 'n_tokens_neox': 933, 'n_words': 379}