<commit_before>// Copyright 2013 <NAME>. All rights reserved.
// license can be found in the LICENSE file (MIT License)
package main

import (
	"fmt"
	"github.com/axw/gollvm/llvm"
	"github.com/bengentil/zlang"
	"github.com/jessevdk/go-flags"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime/pprof"
	"strings"
)

const (
	ResetCode  = "\033[0m" // reset color
	RedCode    = "\033[31m"
	GreenCode  = "\033[32m"
	YellowCode = "\033[33m"
)

type FileType int

const (
	begin_source_files FileType = iota
	FILE_TYPE_C
	FILE_TYPE_ZL
	FILE_TYPE_LL
	FILE_TYPE_BC
	FILE_TYPE_S
	end_source_files

	FILE_TYPE_O
)

var FileExtensions = [...]string{
	FILE_TYPE_C:  ".c",
	FILE_TYPE_ZL: ".zl",
	FILE_TYPE_LL: ".ll",
	FILE_TYPE_BC: ".bc",
	FILE_TYPE_S:  ".s",

	FILE_TYPE_O: ".o",
}

type InputFile struct {
	Name string
	Type FileType
}

func (i *InputFile) String() string {
	return i.Name
}

type Options struct {
	Verbose    bool   `short:"v" long:"verbose" description:"verbose output" default:"false"`
	Debug      bool   `long:"debug" description:"debug output" default:"false"`
	Cpuprofile string `long:"cpuprofile" description:"file to output pprof data" optional:"yes"`
	EmitLLVM   bool   `long:"emit-llvm" description:"print llvm IR and exit" optional:"yes"`
	EmitAST    bool   `long:"emit-ast" description:"print AST JSON representation and exit" optional:"yes"`
	LlcBin     string `long:"llc-bin" description:"llc command to execute" default:"llc"`
	ClangBin   string `long:"clang-bin" description:"clang command to execute" default:"clang"`
	NoLink     bool   `short:"c" description:"don't link (produce object file)" default:"false"`
	Force      bool   `short:"f" description:"force compilation" default:"false"`
	Output     string `short:"o" long:"output" description:"output file" default:"a.out"`
}

var options Options
var inputFiles []*InputFile
var parser = flags.NewParser(&options, flags.Default)

func verbose(format string, v ...interface{}) {
	if options.Verbose {
		fmt.Fprintf(os.Stderr, YellowCode+format+ResetCode+"\n", v...)
	}
}

func stderr(format string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", v...)
}

func (f *InputFile) SetType(t FileType) {
	f.Name = f.ShortName() + FileExtensions[t]
	f.Type = t
}

func (f *InputFile) ShortName() string {
	end := len(f.Name) - len(FileExtensions[f.Type])
	if end > 0 {
		return f.Name[:end]
	} else {
		return f.Name
	}
}

func llc(in *InputFile) error {
	verbose("[llc (%s) %v]", options.LlcBin, in)
	out, err := exec.Command(options.LlcBin, in.Name).CombinedOutput()
	if err != nil {
		return fmt.Errorf("llc error: %v\nOut:%s\n", err, out)
	}
	in.SetType(FILE_TYPE_S)
	return nil
}

func clang_c(in *InputFile) error {
	verbose("[clang_c (%s) %v]", options.ClangBin, in)
	out, err := exec.Command(options.ClangBin, "-c", in.Name).CombinedOutput()
	if err != nil {
		return fmt.Errorf("clang error: %v\nOut:%s\n", err, out)
	}
	in.SetType(FILE_TYPE_O)
	return nil
}

func clang_link(in []*InputFile) error {
	verbose("[clang_link (%s) %v]", options.ClangBin, in)
	args := []string{"-o", options.Output}

	for _, f := range in {
		args = append(args, f.Name)
	}

	out, err := exec.Command(options.ClangBin, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("clang error: %v\nOut:%s\n", err, out)
	}
	return nil
}

func zlc(in *InputFile) error {
	/*
		f, err := os.Open(in.Name)
		if err != nil {
			return err
		}
		defer f.Close()
	*/

	zlang.EnableDebug()

	verbose("[zlc %s]", in.Name)

	source, err := ioutil.ReadFile(in.Name)<commit_msg>Add debug option<commit_after>// Copyright 2013 <NAME>. All rights reserved.
// license can be found in the LICENSE file (MIT License)
package main

import (
	"fmt"
	"github.com/axw/gollvm/llvm"
	"github.com/bengentil/zlang"
	"github.com/jessevdk/go-flags"
	"io/ioutil"
	"os"
	"os/exec"
	"runtime/pprof"
	"strings"
)

const (
	ResetCode  = "\033[0m" // reset color
	RedCode    = "\033[31m"
	GreenCode  = "\033[32m"
	YellowCode = "\033[33m"
)

type FileType int

const (
	begin_source_files FileType = iota
	FILE_TYPE_C
	FILE_TYPE_ZL
	FILE_TYPE_LL
	FILE_TYPE_BC
	FILE_TYPE_S
	end_source_files

	FILE_TYPE_O
)

var FileExtensions = [...]string{
	FILE_TYPE_C:  ".c",
	FILE_TYPE_ZL: ".zl",
	FILE_TYPE_LL: ".ll",
	FILE_TYPE_BC: ".bc",
	FILE_TYPE_S:  ".s",

	FILE_TYPE_O: ".o",
}

type InputFile struct {
	Name string
	Type FileType
}

func (i *InputFile) String() string {
	return i.Name
}

type Options struct {
	Verbose    bool   `short:"v" long:"verbose" description:"verbose output" default:"false"`
	Debug      bool   `long:"debug" description:"debug output" default:"false"`
	Cpuprofile string `long:"cpuprofile" description:"file to output pprof data" optional:"yes"`
	EmitLLVM   bool   `long:"emit-llvm" description:"print llvm IR and exit" optional:"yes"`
	EmitAST    bool   `long:"emit-ast" description:"print AST JSON representation and exit" optional:"yes"`
	LlcBin     string `long:"llc-bin" description:"llc command to execute" default:"llc"`
	ClangBin   string `long:"clang-bin" description:"clang command to execute" default:"clang"`
	NoLink     bool   `short:"c" description:"don't link (produce object file)" default:"false"`
	Force      bool   `short:"f" description:"force compilation" default:"false"`
	Output     string `short:"o" long:"output" description:"output file" default:"a.out"`
}

var options Options
var inputFiles []*InputFile
var parser = flags.NewParser(&options, flags.Default)

func verbose(format string, v ...interface{}) {
	if options.Verbose {
		fmt.Fprintf(os.Stderr, YellowCode+format+ResetCode+"\n", v...)
	}
}

func stderr(format string, v ...interface{}) {
	fmt.Fprintf(os.Stderr, format+"\n", v...)
}

func (f *InputFile) SetType(t FileType) {
	f.Name = f.ShortName() + FileExtensions[t]
	f.Type = t
}

func (f *InputFile) ShortName() string {
	end := len(f.Name) - len(FileExtensions[f.Type])
	if end > 0 {
		return f.Name[:end]
	} else {
		return f.Name
	}
}

func llc(in *InputFile) error {
	verbose("[llc (%s) %v]", options.LlcBin, in)
	out, err := exec.Command(options.LlcBin, in.Name).CombinedOutput()
	if err != nil {
		return fmt.Errorf("llc error: %v\nOut:%s\n", err, out)
	}
	in.SetType(FILE_TYPE_S)
	return nil
}

func clang_c(in *InputFile) error {
	verbose("[clang_c (%s) %v]", options.ClangBin, in)
	out, err := exec.Command(options.ClangBin, "-c", in.Name).CombinedOutput()
	if err != nil {
		return fmt.Errorf("clang error: %v\nOut:%s\n", err, out)
	}
	in.SetType(FILE_TYPE_O)
	return nil
}

func clang_link(in []*InputFile) error {
	verbose("[clang_link (%s) %v]", options.ClangBin, in)
	args := []string{"-o", options.Output}

	for _, f := range in {
		args = append(args, f.Name)
	}

	out, err := exec.Command(options.ClangBin, args...).CombinedOutput()
	if err != nil {
		return fmt.Errorf("clang error: %v\nOut:%s\n", err, out)
	}
	return nil
}

func zlc(in *InputFile) error {
	/*
		f, err := os.Open(in.Name)
		if err != nil {
			return err
		}
		defer f.Close()
	*/

	if options.Debug {
		zlang.EnableDebug()
	}

	verbose("[zlc %s]", in.Name)

	source, err := ioutil.ReadFile(in.Name)
{'fraction_non_alphanumeric': 0.1618374558303887, 'fraction_numerical': 0.006784452296819788, 'mean_word_length': 4.138707334785766, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '453150', 'n_tokens_mistral': 2750, 'n_tokens_neox': 2683, 'n_words': 653}