<commit_before>package main

import (
	"github.com/zarevucky/spl"
	"os"
	"os/exec"
	pathpkg "path"
	"strings"
)

type Application struct {
	Packages map[string]*Package
}

func NewApplication() *Application {
	return &Application{make(map[string]*Package)}
}

// This method loads the builtin "environment" package into the program context
// under a given name. The package contains very basic methods for interfacing
// with operating system (files, output, program args, etc...).
// It only provides the bare minimum of methods necessary for implementing
// a full-featured self-hosting compiler.
func (app *Application) LoadBuiltinEnv(name string) {
	envpkg := NewPackage("<builtin_env>")
	app.Packages[name] = envpkg

	// Takes arbitrary number of arguments, and prints each of them to
	// the standard output, with nothing in between or at the end.
	// It should only be used to print strings and integers if
	// well-defined result is required, but other values may be printed
	// for human debugging.
	envpkg.Methods["print"] = NewBuiltinMethod("env.print", nil)

	// Accepts a filename as the only argument, and returns an
	// array of bytes with the contents of the file. Can fail.
	// FIXME: Document failure modes.
	envpkg.Methods["read_file"] = NewBuiltinMethod("env.read_file", []Type{NewPointerType(NewArrayType(INT_TYPE, -1))})

	// This method just returns the array of command-line arguments
	// for the running program.
	envpkg.Methods["args"] = NewBuiltinMethod("env.args", []Type{NewPointerType(NewArrayType(STRING_TYPE, -1))})

	/*iopkg.Globals["File"] = NewBuiltinType("io.File")
	iopkg.Globals["Open"] = NewBuiltinMethod("io.Open")
	iopkg.Globals["*File.Read"] = NewBuiltinMethod("io.*File.Read")
	iopkg.Globals["*File.Write"] = NewBuiltinMethod("io.*File.Write")
	iopkg.Globals["Dir"] = NewBuiltinType("io.Dir")
	iopkg.Glabals["OpenDir"] = NewBuiltinMethod("io.OpenDir")
	iopkg.Globals["*Dir.Read"] = NewBuiltinMethod("io.*Dir.Read")
	*/
}

func (app *Application) LoadPackage(name string, path string) {
	pkg := NewPackage(path)

	if path[0] == '/' {
		// TODO: PkgDir
		//	path = pathpkg.Join(app.PkgDir, path)
	}

	dir, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	fi, err := dir.Readdir(0)
	if err != nil {
		panic(err)
	}

	for i := range fi {
		if fi[i].IsDir() {
			continue
		}

		if !strings.HasSuffix(fi[i].Name(), ".zs") {
			continue
		}

		filename := pathpkg.Join(path, fi[i].Name())

		var err error

		parser := os.Getenv("ZSPARSER")
		if parser == "" {
			parser = "zsparser"
		}

		cmd := exec.Command(parser)
		cmd.Stdin, err = os.Open(filename)
		if err != nil {
			panic(err)
		}

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			panic(err)
		}

		err = cmd.Start()
		if err != nil {
			panic(err)
		}

		p := spl.NewSeqParser(stdout)

		globals := make([]ASTGlobal, 0)

		for !p.IsEnd() {
			globals = append(globals, ParseASTGlobal(p))
		}

		err = cmd.Wait()
		if err != nil {
			panic(err)
		}

		pkg.LoadFile(app, globals)
	}

	for _, t := range pkg.Types {
		t.Resolve()
	}

	for _, m := range pkg.MethodList {
		m.Resolve()
	}

	for _, m := range pkg.MethodList {
		NewTranscribeContext(m).Transcribe()
	}

	app.Packages[name] = pkg
}<commit_msg>Give better error when parser fails.<commit_after>package main

import (
	"github.com/zarevucky/spl"
	"os"
	"os/exec"
	pathpkg "path"
	"strings"
	"log"
)

type Application struct {
	Packages map[string]*Package
}

func NewApplication() *Application {
	return &Application{make(map[string]*Package)}
}

// This method loads the builtin "environment" package into the program context
// under a given name. The package contains very basic methods for interfacing
// with operating system (files, output, program args, etc...).
// It only provides the bare minimum of methods necessary for implementing
// a full-featured self-hosting compiler.
func (app *Application) LoadBuiltinEnv(name string) {
	envpkg := NewPackage("<builtin_env>")
	app.Packages[name] = envpkg

	// Takes arbitrary number of arguments, and prints each of them to
	// the standard output, with nothing in between or at the end.
	// It should only be used to print strings and integers if
	// well-defined result is required, but other values may be printed
	// for human debugging.
	envpkg.Methods["print"] = NewBuiltinMethod("env.print", nil)

	// Accepts a filename as the only argument, and returns an
	// array of bytes with the contents of the file. Can fail.
	// FIXME: Document failure modes.
	envpkg.Methods["read_file"] = NewBuiltinMethod("env.read_file", []Type{NewPointerType(NewArrayType(INT_TYPE, -1))})

	// This method just returns the array of command-line arguments
	// for the running program.
	envpkg.Methods["args"] = NewBuiltinMethod("env.args", []Type{NewPointerType(NewArrayType(STRING_TYPE, -1))})

	/*iopkg.Globals["File"] = NewBuiltinType("io.File")
	iopkg.Globals["Open"] = NewBuiltinMethod("io.Open")
	iopkg.Globals["*File.Read"] = NewBuiltinMethod("io.*File.Read")
	iopkg.Globals["*File.Write"] = NewBuiltinMethod("io.*File.Write")
	iopkg.Globals["Dir"] = NewBuiltinType("io.Dir")
	iopkg.Glabals["OpenDir"] = NewBuiltinMethod("io.OpenDir")
	iopkg.Globals["*Dir.Read"] = NewBuiltinMethod("io.*Dir.Read")
	*/
}

func (app *Application) LoadPackage(name string, path string) {
	pkg := NewPackage(path)

	if path[0] == '/' {
		// TODO: PkgDir
		//	path = pathpkg.Join(app.PkgDir, path)
	}

	dir, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	fi, err := dir.Readdir(0)
	if err != nil {
		panic(err)
	}

	for i := range fi {
		if fi[i].IsDir() {
			continue
		}

		if !strings.HasSuffix(fi[i].Name(), ".zs") {
			continue
		}

		filename := pathpkg.Join(path, fi[i].Name())

		var err error

		parser := os.Getenv("ZSPARSER")
		if parser == "" {
			parser = "zsparser"
		}

		cmd := exec.Command(parser)
		cmd.Stdin, err = os.Open(filename)
		if err != nil {
			log.Fatalf("Cannot open file %s: %s", filename, err)
		}
		
		// TODO: Get rid of panics.

		stdout, err := cmd.StdoutPipe()
		if err != nil {
			panic(err)
		}

		err = cmd.Start()
		if err != nil {
			panic(err)
		}

		p := spl.NewSeqParser(stdout)

		globals := make([]ASTGlobal, 0)

		for !p.IsEnd() {
			globals = append(globals, ParseASTGlobal(p))
		}

		err = cmd.Wait()
		if err != nil {
			log.Fatalf("Parser failed of file %s: %s", filename, err)
		}

		pkg.LoadFile(app, globals)
	}

	for _, t := range pkg.Types {
		t.Resolve()
	}

	for _, m := range pkg.MethodList {
		m.Resolve()
	}

	for _, m := range pkg.MethodList {
		NewTranscribeContext(m).Transcribe()
	}

	app.Packages[name] = pkg
}
{'fraction_non_alphanumeric': 0.14990886998784933, 'fraction_numerical': 0.0015188335358444715, 'mean_word_length': 3.730603448275862, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6243693', 'n_tokens_mistral': 2448, 'n_tokens_neox': 2283, 'n_words': 610}