<commit_before>package main

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const (
	diskIndexDB = "./file-index.db"
	breakPoint  = 10000
)

var db *sql.DB

func main() {
	//init
	// os.Remove(diskIndexDB)
	// os.Create("indexer.txt")
	// os.Create("copy.txt")

	bootDatabase()

	// diskA := os.Args[1]
	diskB := os.Args[2]
	dest := os.Args[3]

	// getFilesFromFolder(true, diskA)
	getFilesFromFolder(false, diskB)

	fmt.Println("--- FINISHED INDEXING ---")
	fmt.Println("--- STARTING COPY ---")

	files := findMissingFiles()
	copyFiles(dest, files)

	db.Close()

	fmt.Println("--- FINISHED ---")

}

func bootDatabase() {
	var err error

	//We're going to store disk A information in a SQLite database
	db, err = sql.Open("sqlite3", diskIndexDB)
	if err != nil {
		log.Fatal("Could not create the SQLite database", err)
	}

	_, err = db.Exec(`
	CREATE TABLE file (
		id INTEGER PRIMARY KEY AUTOINCREMENT,
		origin BOOLEAN,
		hash VARCHAR(64),
        path VARCHAR(2048)
    );`)

	if err != nil {
		log.Fatal("DB-TABLE CREATE: fail ", err)
	}
}

func insertFile(origin bool, filePath string) {
	//Not really sure what to do if there is an error, so let's log and quit to be safe. We don't want to mis files
	stmt, err := db.Prepare("INSERT INTO file(origin, hash, path) VALUES (?,?,?)")
	if err != nil {
		redoLog("indexer", filePath)
		return
	}

	md5, err := getMD5FromFile(filePath)
	if err != nil {
		redoLog("indexer", filePath)
		return
	}

	_, err = stmt.Exec(origin, md5, filePath)
	if err != nil {
		redoLog("indexer", filePath)
		return
	}
}

func getFilesFromFolder(origin bool, searchDir string) {
	fmt.Println("Indexing folder: " + searchDir)

	var count int64 = 0
<commit_msg>and do not create table<commit_after>package main

import (
	"crypto/md5"
	"database/sql"
	"fmt"
	"io"
	"log"
	"os"
	"path/filepath"
	"strings"

	_ "github.com/mattn/go-sqlite3"
)

const (
	diskIndexDB = "./file-index.db"
	breakPoint  = 10000
)

var db *sql.DB

func main() {
	//init
	// os.Remove(diskIndexDB)
	// os.Create("indexer.txt")
	// os.Create("copy.txt")

	bootDatabase()

	// diskA := os.Args[1]
	diskB := os.Args[2]
	dest := os.Args[3]

	// getFilesFromFolder(true, diskA)
	getFilesFromFolder(false, diskB)

	fmt.Println("--- FINISHED INDEXING ---")
	fmt.Println("--- STARTING COPY ---")

	files := findMissingFiles()
	copyFiles(dest, files)

	db.Close()

	fmt.Println("--- FINISHED ---")

}

func bootDatabase() {
	var err error

	//We're going to store disk A information in a SQLite database
	db, err = sql.Open("sqlite3", diskIndexDB)
	if err != nil {
		log.Fatal("Could not create the SQLite database", err)
	}

	// _, err = db.Exec(`
	// CREATE TABLE file (
	// 	id INTEGER PRIMARY KEY AUTOINCREMENT,
	// 	origin BOOLEAN,
	// 	hash VARCHAR(64),
	//     path VARCHAR(2048)
	// );`)

	// if err != nil {
	// 	log.Fatal("DB-TABLE CREATE: fail ", err)
	// }
}

func insertFile(origin bool, filePath string) {
	//Not really sure what to do if there is an error, so let's log and quit to be safe. We don't want to mis files
	stmt, err := db.Prepare("INSERT INTO file(origin, hash, path) VALUES (?,?,?)")
	if err != nil {
		redoLog("indexer", filePath)
		return
	}

	md5, err := getMD5FromFile(filePath)
	if err != nil {
		redoLog("indexer", filePath)
		return
	}

	_, err = stmt.Exec(origin, md5, filePath)
	if err != nil {
		redoLog("indexer", filePath)
		return
	}
}

func getFilesFromFolder(origin bool, searchDir string) {
	fmt.Println("Indexing folder: " + searchDir)

	var count int64 = 0

{'fraction_non_alphanumeric': 0.15857063093243998, 'fraction_numerical': 0.012841987716359575, 'mean_word_length': 3.2705601907032182, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '148180', 'n_tokens_mistral': 1464, 'n_tokens_neox': 1364, 'n_words': 360}