<commit_before>package main

import "io/ioutil"
import "path"
import "path/filepath"
import "errors"
import "fmt"
import "strings"
import "strconv"
import "os"

type Assemblable map[string]Instruction

type Instruction struct {
	Data, Path string
	Indent, Indented, Args int
	
	Indentation int
	Else *Instruction
	Function func(args []string) string
}

func is(s string, ns ...int) Instruction {
	var args, ind, indent int
	if len(ns) > 0 {
		args = ns[0]
	}
	if len(ns) > 1 {
		indent = ns[1]
	}
	if len(ns) > 2 {
		ind = ns[2]
	}
	return Instruction{Data:s, Args:args, Indent:indent, Indentation:ind}
}

func Reserved() Instruction {
	return Instruction{}
}

func (asm Assemblable) Header() []byte {
	b, err := asm.Assemble("HEADER", []string{asm["NAME"].Data})
	if err != nil {
		panic(err.Error())
	}
	return b
}

func (asm Assemblable) Indentation(n ...int) string {
	if _, ok := asm["INDENT"]; !ok {
		return ""
	}
	if len(n) > 0 {
		if int(asm["INDENT"].Indent)+n[0] < 0 {
			return ""
		}
		return strings.Repeat("\t", int(asm["INDENT"].Indent)+n[0])
	} else {
		return strings.Repeat("\t", int(asm["INDENT"].Indent))
	}
}

func (asm Assemblable) SetFileName(name string) {
	if _, err := os.Stat(path.Dir(name)+asm["FILE"].Path); os.IsNotExist(err) {
		ioutil.WriteFile(path.Dir(name)+asm["FILE"].Path, []byte(asm["FILE"].Data), 0600)
	}
	var extension = filepath.Ext(name)
	name = filepath.Base(name[0:len(name)-len(extension)])
	asm["NAME"] = Instruction{Data:path.Base(name)}
}

func (asm Assemblable) Footer() []byte {
	b, _ := asm.Assemble("FOOTER", nil)
	return b
}

func (asm Assemblable) Assemble(command string, args []string) ([]byte, error) {
	instruction, ok := asm[command]
	if !ok || instruction.Data  == "" {
		return nil, errors.New("Unrecognised command! "+command)
	}
	if command == "NUMBER" || command == "SIZE" || command == "STRING" {
		return []byte(fmt.Sprintf(instruction.Data, args[0])), nil
	}
	if command == "ERRORS" {
		return []byte(fmt.Sprintf(instruction.Data)), nil
	}
	for i, arg := range args {
		if len(arg) == 0 {
			break
		}
	
		var b []byte
		if _, err := strconv.Atoi(arg); err == nil {
			b, err = asm.Assemble("NUMBER", []string{arg})
			if err != nil {
				panic(err.Error())
			}
			args[i] = string(b)
			continue
		}
		<commit_msg>Add embeddable native code instructions.<commit_after>package main

import "io/ioutil"
import "path"
import "path/filepath"
import "errors"
import "fmt"
import "strings"
import "strconv"
import "os"

type Assemblable map[string]Instruction

type Instruction struct {
	Data, Path string
	Indent, Indented, Args int
	
	All bool
	
	Indentation int
	Else *Instruction
	Function func(args []string) string
}

func is(s string, ns ...int) Instruction {
	var args, ind, indent int
	if len(ns) > 0 {
		args = ns[0]
	}
	if len(ns) > 1 {
		indent = ns[1]
	}
	if len(ns) > 2 {
		ind = ns[2]
	}
	return Instruction{Data:s, Args:args, Indent:indent, Indentation:ind}
}

func Reserved() Instruction {
	return Instruction{}
}

func (asm Assemblable) Header() []byte {
	b, err := asm.Assemble("HEADER", []string{asm["NAME"].Data})
	if err != nil {
		panic(err.Error())
	}
	return b
}

func (asm Assemblable) Indentation(n ...int) string {
	if _, ok := asm["INDENT"]; !ok {
		return ""
	}
	if len(n) > 0 {
		if int(asm["INDENT"].Indent)+n[0] < 0 {
			return ""
		}
		return strings.Repeat("\t", int(asm["INDENT"].Indent)+n[0])
	} else {
		return strings.Repeat("\t", int(asm["INDENT"].Indent))
	}
}

func (asm Assemblable) SetFileName(name string) {
	if _, err := os.Stat(path.Dir(name)+asm["FILE"].Path); os.IsNotExist(err) {
		ioutil.WriteFile(path.Dir(name)+asm["FILE"].Path, []byte(asm["FILE"].Data), 0600)
	}
	var extension = filepath.Ext(name)
	name = filepath.Base(name[0:len(name)-len(extension)])
	asm["NAME"] = Instruction{Data:path.Base(name)}
}

func (asm Assemblable) Footer() []byte {
	b, _ := asm.Assemble("FOOTER", nil)
	return b
}

var Languages = map[string]bool {
	"JAVA":true,
	"GO":true,
	"PYTHON":true,
}

func (asm Assemblable) Assemble(command string, args []string) ([]byte, error) {
	instruction, ok := asm[command]
	if !ok {
		if Languages[command] {
			return []byte(""), nil
		}
		return nil, errors.New("Unrecognised command! "+command)
	}
	if instruction.All {
		return []byte(strings.Join(args, " ")+"\n"), nil
	}
	if instruction.Data  == "" {
		return nil, errors.New("Bad command! "+command)
	} 
	if command == "NUMBER" || command == "SIZE" || command == "STRING" {
		return []byte(fmt.Sprintf(instruction.Data, args[0])), nil
	}
	if command == "ERRORS" {
		return []byte(fmt.Sprintf(instruction.Data)), nil
	}
	for i, arg := range args {
		if len(arg) == 0 {
			break
		}
	
		var b []byte
		if _, err := strconv.Atoi(arg); err == nil {
			b, err = asm.Assemble("NUMBER", []string{arg})
			if err != nil {
				panic(err.Error())
			}
			args[i] = string(b)
			continue
		}
		
{'fraction_non_alphanumeric': 0.1818741029321304, 'fraction_numerical': 0.006971498872257536, 'mean_word_length': 4.044467425025853, 'pattern_counts': {'":': 3, '<': 5, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '7014256', 'n_tokens_mistral': 1965, 'n_tokens_neox': 1845, 'n_words': 447}