<commit_before>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

const headerData = `
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
`

func producer(filePath string, readCh chan string) {
	file, err := os.Open(filePath)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	reader := bufio.NewReader(file)
	for {
		line, err := reader.ReadString('\n')
		if err != nil {
			break
		}
		readCh <- line
	}
	close(readCh)
}

func consumer(readCh, tableNameCh, tableSchemeCh, tableDataCh chan string) {
	onTableScheme := false
	onTableData := false
	for line := range readCh {
		if strings.Contains(line, "Table structure for table") {
			onTableScheme = true
			onTableData = false
			tableName := strings.Replace(line, "-- Table structure for table ", "", 1)
			tableNameCh <- strings.TrimSpace(strings.Replace(tableName, "`", "", -1))
			tableSchemeCh <- line
		} else if strings.Contains(line, "LOCK TABLES `") {
			onTableData = true
			onTableScheme = false
			tableDataCh <- line
		} else {
			if onTableScheme {
				tableSchemeCh <- line
			}
			if onTableData {
				tableDataCh <- line
			}

			if strings.Contains(line, "-- Dumping data for table") {
				onTableScheme = false
				tableSchemeCh <- "SENTINEL_STRING"
			} else if strings.Contains(line, "UNLOCK TABLES;") {
				onTableData = false
				tableDataCh <- "SENTINEL_STRING"
			}
		}
	}
	close(tableNameCh)
	close(tableDataCh)
	close(tableSchemeCh)
}

func writer(outputDir string, tableNameCh, tableSchemeCh, tableDataCh chan string, doneCh chan bool) {
	os.Mkdir(outputDir, os.ModePerm)

	for tableName := range tableNameCh {
		fmt.Printf("%s\n", tableName)
		tablePath := filepath.Join(outputDir, tableName+".sql")
		tableFile, _ := os.Create(tablePath)

		for tableData := range tableSchemeCh {
			if tableData == "SENTINEL_STRING" {
				break
			}
			tableFile.WriteString(tableData)
		}

		for tableData := range tableDataCh {
			if tableData == "SENTINEL_STRING" {
				break
			}
			if tableName != "default_api_logs" && tableName != "default_ci_sessions" {
				tableFile.WriteString(tableData)
			}
		}
		tableFile.Close()
	}
	doneCh <- true
}

func combineFiles(filePath, outputDir string) {
	combineFile, _ := os.Create(filePath)<commit_msg>Moved sentinel value into a const<commit_after>package main

import (
	"bufio"
	"flag"
	"fmt"
	"io/ioutil"
	"os"
	"path/filepath"
	"strings"
)

const (
	// Used to tell the
	sentinelString = "****SENTINEL-STRING****"
	headerData     = `
/*!40101 SET @OLD_CHARACTER_SET_CLIENT=@@CHARACTER_SET_CLIENT */;
/*!40101 SET @OLD_CHARACTER_SET_RESULTS=@@CHARACTER_SET_RESULTS */;
/*!40101 SET @OLD_COLLATION_CONNECTION=@@COLLATION_CONNECTION */;
/*!40101 SET NAMES utf8 */;
/*!40103 SET @OLD_TIME_ZONE=@@TIME_ZONE */;
/*!40103 SET TIME_ZONE='+00:00' */;
/*!40014 SET @OLD_UNIQUE_CHECKS=@@UNIQUE_CHECKS, UNIQUE_CHECKS=0 */;
/*!40014 SET @OLD_FOREIGN_KEY_CHECKS=@@FOREIGN_KEY_CHECKS, FOREIGN_KEY_CHECKS=0 */;
/*!40101 SET @OLD_SQL_MODE=@@SQL_MODE, SQL_MODE='NO_AUTO_VALUE_ON_ZERO' */;
/*!40111 SET @OLD_SQL_NOTES=@@SQL_NOTES, SQL_NOTES=0 */;
`
)

func producer(filePath string, readCh chan string) {
	file, err := os.Open(filePath)
	if err != nil {
		panic(err)
	}
	defer file.Close()

	r := bufio.NewReader(file)
	for line, err := r.ReadString('\n'); err == nil; line, err = r.ReadString('\n') {
		readCh <- line
	}
	close(readCh)
}

func consumer(readCh, tableNameCh, tableSchemeCh, tableDataCh chan string) {
	onTableScheme, onTableData := false, false
	for line := range readCh {
		if strings.Contains(line, "Table structure for table") {
			onTableScheme, onTableData = true, false
			tableName := strings.Replace(line, "-- Table structure for table ", "", 1)
			tableNameCh <- strings.TrimSpace(strings.Replace(tableName, "`", "", -1))
			tableSchemeCh <- line
		} else if strings.Contains(line, "LOCK TABLES `") {
			onTableData, onTableScheme = true, false
			tableDataCh <- line
		} else {
			if onTableScheme {
				tableSchemeCh <- line
			}
			if onTableData {
				tableDataCh <- line
			}

			if strings.Contains(line, "-- Dumping data for table") {
				onTableScheme = false
				tableSchemeCh <- sentinelString
			} else if strings.Contains(line, "UNLOCK TABLES;") {
				onTableData = false
				tableDataCh <- sentinelString
			}
		}
	}
	close(tableNameCh)
	close(tableDataCh)
	close(tableSchemeCh)
}

func writer(outputDir string, tableNameCh, tableSchemeCh, tableDataCh chan string, doneCh chan bool) {
	os.Mkdir(outputDir, os.ModePerm)

	for tableName := range tableNameCh {
		fmt.Printf("extracting table: %s\n", tableName)
		tablePath := filepath.Join(outputDir, tableName+".sql")
		tableFile, _ := os.Create(tablePath)

		for tableData := range tableSchemeCh {
			if tableData == sentinelString {
				break
			}
			tableFile.WriteString(tableData)
		}

		for tableData := range tableDataCh {
			if tableData == sentinelString {
				break
			}
			if tableName != "default_api_logs" && tableName != "default_ci_sessions" {
				tableFile.WriteString(tableData)
			}
		}
		tableFile.Close()
	}
	doneCh <- true
}

func combineFiles(filePath, outputDir string) {
	combineFile, _ := os.Create(filePath)
{'fraction_non_alphanumeric': 0.13810607386662, 'fraction_numerical': 0.021004726063364256, 'mean_word_length': 4.189827429609446, 'pattern_counts': {'":': 0, '<': 21, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5170809', 'n_tokens_mistral': 2352, 'n_tokens_neox': 2066, 'n_words': 433}