<commit_before>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package sqlite3

import (
	"database/sql"
	"fmt"
	"github.com/conformal/btcdb"
	"github.com/conformal/btcutil"
	"github.com/conformal/btcwire"
	"github.com/conformal/seelog"
	_ "github.com/mattn/go-sqlite3"
	"os"
	"sync"
)

const (
	dbVersion     int = 2
	dbMaxTransCnt     = 20000
	dbMaxTransMem     = 64 * 1024 * 1024 // 64 MB
)

const (
	blkInsertSha = iota
	blkFetchSha
	blkExistsSha
	blkFetchIdx
	blkFetchIdxList
)

const (
	txInsertStmt = iota
	txFetchUsedByShaStmt
	txFetchLocationByShaStmt
	txtmpInsertStmt
	txtmpFetchUsedByShaStmt
	txtmpFetchLocationByShaStmt
	txMigrateCopy
	txMigrateClear
	txMigratePrep
	txMigrateFinish
	txMigrateCount
	txPragmaVacuumOn
	txPragmaVacuumOff
	txVacuum
)

var blkqueries []string = []string{
	blkInsertSha:    "INSERT INTO block (key, pver, data) VALUES(?, ?, ?);",
	blkFetchSha:     "SELECT pver, data, blockid FROM block WHERE key = ?;",
	blkExistsSha:    "SELECT pver FROM block WHERE key = ?;",
	blkFetchIdx:     "SELECT key FROM block WHERE blockid = ?;",
	blkFetchIdxList: "SELECT key FROM block WHERE blockid >= ? AND blockid < ? ORDER BY blockid ASC LIMIT 500;",
}

var txqueries []string = []string{
	txInsertStmt:                "INSERT INTO tx (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);",
	txFetchUsedByShaStmt:        "SELECT data FROM tx WHERE key = ?;",
	txFetchLocationByShaStmt:    "SELECT blockid, txoff, txlen FROM tx WHERE key = ?;",
	txtmpInsertStmt:             "INSERT INTO txtmp (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);",
	txtmpFetchUsedByShaStmt:     "SELECT data FROM txtmp WHERE key = ?;",
	txtmpFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM txtmp WHERE key = ?;",
	txMigrateCopy:               "INSERT INTO tx (key, blockid, txoff, txlen, data) SELECT key, blockid, txoff, txlen, data FROM txtmp;",
	txMigrateClear:              "DELETE from txtmp;",
	txMigratePrep:               "DROP index uniquetx;",
	txMigrateFinish:             "CREATE UNIQUE INDEX IF NOT EXISTS uniquetx ON tx (key);",
	txMigrateCount:              "SELECT COUNT(*) FROM txtmp;",
	txPragmaVacuumOn:            "PRAGMA auto_vacuum = FULL;",
	txPragmaVacuumOff:           "PRAGMA auto_vacuum = NONE;",
	txVacuum:                    "VACUUM;",
}

var log seelog.LoggerInterface = seelog.Disabled

type tBlockInsertData struct {
	sha  btcwire.ShaHash
	pver uint32
	buf  []byte
}
type tTxInsertData struct {
	txsha   *btcwire.ShaHash
	blockid int64
	txoff   int
	txlen   int
	usedbuf []byte
}

type txState struct {
	tx           *sql.Tx<commit_msg>Only try to drop the uniquetx index if it exists.<commit_after>// Copyright (c) 2013 Conformal Systems LLC.
// Use of this source code is governed by an ISC
// license that can be found in the LICENSE file.

package sqlite3

import (
	"database/sql"
	"fmt"
	"github.com/conformal/btcdb"
	"github.com/conformal/btcutil"
	"github.com/conformal/btcwire"
	"github.com/conformal/seelog"
	_ "github.com/mattn/go-sqlite3"
	"os"
	"sync"
)

const (
	dbVersion     int = 2
	dbMaxTransCnt     = 20000
	dbMaxTransMem     = 64 * 1024 * 1024 // 64 MB
)

const (
	blkInsertSha = iota
	blkFetchSha
	blkExistsSha
	blkFetchIdx
	blkFetchIdxList
)

const (
	txInsertStmt = iota
	txFetchUsedByShaStmt
	txFetchLocationByShaStmt
	txtmpInsertStmt
	txtmpFetchUsedByShaStmt
	txtmpFetchLocationByShaStmt
	txMigrateCopy
	txMigrateClear
	txMigratePrep
	txMigrateFinish
	txMigrateCount
	txPragmaVacuumOn
	txPragmaVacuumOff
	txVacuum
)

var blkqueries []string = []string{
	blkInsertSha:    "INSERT INTO block (key, pver, data) VALUES(?, ?, ?);",
	blkFetchSha:     "SELECT pver, data, blockid FROM block WHERE key = ?;",
	blkExistsSha:    "SELECT pver FROM block WHERE key = ?;",
	blkFetchIdx:     "SELECT key FROM block WHERE blockid = ?;",
	blkFetchIdxList: "SELECT key FROM block WHERE blockid >= ? AND blockid < ? ORDER BY blockid ASC LIMIT 500;",
}

var txqueries []string = []string{
	txInsertStmt:                "INSERT INTO tx (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);",
	txFetchUsedByShaStmt:        "SELECT data FROM tx WHERE key = ?;",
	txFetchLocationByShaStmt:    "SELECT blockid, txoff, txlen FROM tx WHERE key = ?;",
	txtmpInsertStmt:             "INSERT INTO txtmp (key, blockid, txoff, txlen, data) VALUES(?, ?, ?, ?, ?);",
	txtmpFetchUsedByShaStmt:     "SELECT data FROM txtmp WHERE key = ?;",
	txtmpFetchLocationByShaStmt: "SELECT blockid, txoff, txlen FROM txtmp WHERE key = ?;",
	txMigrateCopy:               "INSERT INTO tx (key, blockid, txoff, txlen, data) SELECT key, blockid, txoff, txlen, data FROM txtmp;",
	txMigrateClear:              "DELETE from txtmp;",
	txMigratePrep:               "DROP index IF EXISTS uniquetx;",
	txMigrateFinish:             "CREATE UNIQUE INDEX IF NOT EXISTS uniquetx ON tx (key);",
	txMigrateCount:              "SELECT COUNT(*) FROM txtmp;",
	txPragmaVacuumOn:            "PRAGMA auto_vacuum = FULL;",
	txPragmaVacuumOff:           "PRAGMA auto_vacuum = NONE;",
	txVacuum:                    "VACUUM;",
}

var log seelog.LoggerInterface = seelog.Disabled

type tBlockInsertData struct {
	sha  btcwire.ShaHash
	pver uint32
	buf  []byte
}
type tTxInsertData struct {
	txsha   *btcwire.ShaHash
	blockid int64
	txoff   int
	txlen   int
	usedbuf []byte
}

type txState struct {
	tx           *sql.Tx
{'fraction_non_alphanumeric': 0.10254050073637702, 'fraction_numerical': 0.011413843888070692, 'mean_word_length': 3.39919028340081, 'pattern_counts': {'":': 0, '<': 5, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1601950', 'n_tokens_mistral': 2084, 'n_tokens_neox': 2004, 'n_words': 532}