<commit_before>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/user"
	"runtime"

	. "bind.ch/blkidx"

	_ "github.com/mattn/go-sqlite3"
)

var (
	flagDb          *string
	flagConcurrency = flag.Int("c", 1, "concurrency")
	logger          = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds)
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	var db string
	user, err := user.Current()
	if err == nil {
		db = user.HomeDir + string(os.PathSeparator) + ".blkidx.sqlite3"
	}
	flagDb = flag.String("db", db, "sqlite database file to store")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `
%s [options] command <path>

commands:

  index [path...]      add (or update) a path to the index.
                     an empty path mean.

  dups               show files with duplicate checksums


options:
`, os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	args := flag.Args()
	if *flagDb == "" || len(args) == 0 {
		errUsage()
	}

	idx, dbCloser, err := openDbIndex(*flagDb)
	if err != nil {
		logger.Printf("failed to open the sqlite3 database: %v", err)
		os.Exit(2)
	}
	defer dbCloser.Close()

	switch args[0] {
	case "index":
		if len(args) == 1 {
			index(idx, []string{"."})
		} else {
			index(idx, args[1:])
		}
	//case "dups":
	//	dups(idx)
	default:
		flag.Usage()
		os.Exit(1)
	}
}

func errUsage() {
	flag.Usage()
	os.Exit(1)
}

func openDbIndex(dbUrl string) (Index, io.Closer, error) {
	// TODO: doesn'nt work, see comment below
	//dbUrl := "file:" + *flagDb + "?cache=shared&mode=rwc"
	db, err := sql.Open("sqlite3", dbUrl)
	if err != nil {
		return nil, nil, err
	}
	idx, err := NewSqlIndex(db)
	if err != nil {
		db.Close()
		return nil, nil, err<commit_msg>cmd/blkidx: ensure that the database is closed on program exit<commit_after>package main

import (
	"database/sql"
	"flag"
	"fmt"
	"io"
	"log"
	"os"
	"os/user"
	"runtime"

	. "bind.ch/blkidx"

	_ "github.com/mattn/go-sqlite3"
)

var (
	flagDb          *string
	flagConcurrency = flag.Int("c", 1, "concurrency")
	logger          = log.New(os.Stderr, "", log.Ldate|log.Ltime|log.Lmicroseconds)
)

func init() {
	runtime.GOMAXPROCS(runtime.NumCPU())

	var db string
	user, err := user.Current()
	if err == nil {
		db = user.HomeDir + string(os.PathSeparator) + ".blkidx.sqlite3"
	}
	flagDb = flag.String("db", db, "sqlite database file to store")

	flag.Usage = func() {
		fmt.Fprintf(os.Stderr, `
%s [options] command <path>

commands:

  index [path...]      add (or update) a path to the index.
                     an empty path mean.

  dups               show files with duplicate checksums


options:
`, os.Args[0])
		flag.PrintDefaults()
	}
}

func main() {
	flag.Parse()
	args := flag.Args()
	if *flagDb == "" || len(args) == 0 {
		errUsage(nil)
	}

	idx, dbCloser, err := openDbIndex(*flagDb)
	if err != nil {
		logger.Printf("failed to open the sqlite3 database: %v", err)
		os.Exit(2)
	}
	defer dbCloser.Close()

	switch args[0] {
	case "index":
		if len(args) == 1 {
			index(idx, []string{"."})
		} else {
			index(idx, args[1:])
		}

	case "dups":
		dups(idx)

	default:
		errUsage(dbCloser)
	}
}

func errUsage(c io.Closer) {
	if c != nil {
		c.Close()
	}
	flag.Usage()
	os.Exit(1)
}

func openDbIndex(dbUrl string) (Index, io.Closer, error) {
	// TODO: doesn'nt work, see comment below
	//dbUrl := "file:" + *flagDb + "?cache=shared&mode=rwc"
	db, err := sql.Open("sqlite3", dbUrl)
	if err != nil {
		return nil, nil, err
	}
	idx, err := NewSqlIndex(db)
	if err != nil {
		db.Close()
		return nil, nil, err
{'fraction_non_alphanumeric': 0.17154105736782901, 'fraction_numerical': 0.00703037120359955, 'mean_word_length': 3.005630630630631, 'pattern_counts': {'":': 4, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3260488', 'n_tokens_mistral': 1485, 'n_tokens_neox': 1415, 'n_words': 344}