<commit_before>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/soh335/schemalex"
)

var (
	before       = flag.String("before", "", "file of before schema")
	after        = flag.String("after", "", "file of after schema")
	errorMarker  = flag.String("error-marker", "___", "marker of parse error position")
	errorContext = flag.Int("error-context", 20, "before, after context position of parse error")
)

func main() {
	flag.Parse()

	if err := _main(); err != nil {
		log.Fatal(err)
	}
}

func _main() error {
	b, err := ioutil.ReadFile(*before)
	if err != nil {
		return err
	}

	a, err := ioutil.ReadFile(*after)
	if err != nil {
		return err
	}

	p := schemalex.NewParser(string(b))
	p.ErrorMarker = *errorMarker
	p.ErrorContext = *errorContext

	beforeStmts, err := p.Parse()
	if err != nil {
		return fmt.Errorf("file:%s error:%s", *before, err)
	}

	p = schemalex.NewParser(string(a))
	p.ErrorMarker = *errorMarker
	p.ErrorContext = *errorContext

	afterStmts, err := p.Parse()
	if err != nil {
		return fmt.Errorf("file:%s error:%s", *after, err)
	}

	d := &schemalex.Differ{filterCreateTableStatement(beforeStmts), filterCreateTableStatement(afterStmts)}
	d.WriteDiffWithTransaction(os.Stdout)

	return nil
}
<commit_msg>pass directory before, after sql<commit_after>package main

import (
	"flag"
	"fmt"
	"io/ioutil"
	"log"
	"os"

	"github.com/soh335/schemalex"
)

var (
	errorMarker  = flag.String("error-marker", "___", "marker of parse error position")
	errorContext = flag.Int("error-context", 20, "before, after context position of parse error")
)

func main() {
	flag.Parse()
	args := flag.Args()

	if len(args) != 2 {
		log.Fatalf("should call schemalex <options> /path/to/before /path/to/after")
	}

	if err := _main(args[0], args[1]); err != nil {
		log.Fatal(err)
	}
}

func _main(before, after string) error {
	b, err := ioutil.ReadFile(before)
	if err != nil {
		return err
	}

	a, err := ioutil.ReadFile(after)
	if err != nil {
		return err
	}

	p := schemalex.NewParser(string(b))
	p.ErrorMarker = *errorMarker
	p.ErrorContext = *errorContext

	beforeStmts, err := p.Parse()
	if err != nil {
		return fmt.Errorf("file:%s error:%s", before, err)
	}

	p = schemalex.NewParser(string(a))
	p.ErrorMarker = *errorMarker
	p.ErrorContext = *errorContext

	afterStmts, err := p.Parse()
	if err != nil {
		return fmt.Errorf("file:%s error:%s", after, err)
	}

	d := &schemalex.Differ{filterCreateTableStatement(beforeStmts), filterCreateTableStatement(afterStmts)}
	d.WriteDiffWithTransaction(os.Stdout)

	return nil
}

{'fraction_non_alphanumeric': 0.1579563182527301, 'fraction_numerical': 0.005070202808112325, 'mean_word_length': 4.150602409638554, 'pattern_counts': {'":': 0, '<': 4, '<?xml version=': 0, '>': 4, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5285508', 'n_tokens_mistral': 992, 'n_tokens_neox': 942, 'n_words': 220}