<commit_before>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/afrase/mysqldumpsplit/msds"
)

type config struct {
	InputFile       string
	OutputPath      string
	CombineFilePath string
	Combine         bool
	Skip            msds.SkipTables
}

func openFile(path string) *os.File {
	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	info, _ := file.Stat()
	fmt.Printf("Original file size %s\n", msds.StringifyFileSize(info.Size()))

	return file
}

func parseFlags() *config {
	conf := new(config)

	flag.StringVar(&conf.InputFile, "i", "", "The file to read from")
	flag.StringVar(&conf.OutputPath, "o", "output", "The output path ")

	flag.Var(&conf.Skip, "skipData", "Comma separated list of tables you don't want the data for")

	flag.BoolVar(&conf.Combine, "combine", false, "Combine all tables into a single file")
	flag.StringVar(&conf.CombineFilePath, "combineFile", "dumpfile.sql",
		"The path to output a single SQL file\n\tOnly used if combine flag is set")

	flag.Parse()
	return conf
}

func main() {
	conf := parseFlags()

	readCh := make(chan string)
	tableNameCh := make(chan string)
	tableDataCh := make(chan string)
	tableSchemeCh := make(chan string)
	doneCh := make(chan bool)

	fmt.Printf("Outputing all tables to %s\n", conf.OutputPath)
	if len(conf.Skip) > 0 {
		fmt.Printf("Skiping data from tables %s\n", strings.Join(conf.Skip, ", "))
	}

	file := openFile(conf.InputFile)

	fmt.Printf("Begin processing %s\n\n", conf.InputFile)
	// create a pipeline of goroutines
	go msds.Producer(file, readCh)<commit_msg>Check if file exists before trying to open it<commit_after>package main

import (
	"flag"
	"fmt"
	"os"
	"strings"

	"github.com/afrase/mysqldumpsplit/msds"
)

type config struct {
	InputFile       string
	OutputPath      string
	CombineFilePath string
	Combine         bool
	Skip            msds.SkipTables
}

func openFile(path string) *os.File {
	if _, err := os.Stat(path); os.IsNotExist(err) {
		fmt.Fprintf(os.Stderr, "The file '%s' does not exist.\n", path)
		os.Exit(1)
	}

	file, err := os.Open(path)
	if err != nil {
		panic(err)
	}

	info, _ := file.Stat()
	fmt.Printf("Original file size %s\n", msds.StringifyFileSize(info.Size()))

	return file
}

func parseFlags() *config {
	conf := new(config)

	flag.StringVar(&conf.InputFile, "i", "", "The file to read from")
	flag.StringVar(&conf.OutputPath, "o", "output", "The output path ")

	flag.Var(&conf.Skip, "skipData", "Comma separated list of tables you don't want the data for")

	flag.BoolVar(&conf.Combine, "combine", false, "Combine all tables into a single file")
	flag.StringVar(&conf.CombineFilePath, "combineFile", "dumpfile.sql",
		"The path to output a single SQL file\n\tOnly used if combine flag is set")

	flag.Parse()
	return conf
}

func main() {
	conf := parseFlags()

	readCh := make(chan string)
	tableNameCh := make(chan string)
	tableDataCh := make(chan string)
	tableSchemeCh := make(chan string)
	doneCh := make(chan bool)

	file := openFile(conf.InputFile)

	fmt.Printf("Outputing all tables to %s\n", conf.OutputPath)
	if len(conf.Skip) > 0 {
		fmt.Printf("Skiping data from tables %s\n", strings.Join(conf.Skip, ", "))
	}

	fmt.Printf("Begin processing %s\n\n", conf.InputFile)
	// create a pipeline of goroutines
	go msds.Producer(file, readCh)
{'fraction_non_alphanumeric': 0.14177521963041503, 'fraction_numerical': 0.0009088155104513784, 'mean_word_length': 3.987915407854985, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '384365', 'n_tokens_mistral': 1195, 'n_tokens_neox': 1164, 'n_words': 320}