<commit_before>package cache

import (
	"database/sql"
	"fmt"
	"time"

	"gitgud.io/softashell/comfy-translator/translator"
	_ "github.com/mattn/go-sqlite3" // Sql driver
	log "github.com/sirupsen/logrus"
)

type translationError int

const (
	errorNone           translationError = iota // Everything is fine
	errorMinor                                  // Connection timed out or something like that
	errorBadTranslation                         // Returned really bad translation
)

const (
	cleanupInterval = (24 * time.Hour) * 7
)

type Cache struct {
	db *sql.DB
}

type Item struct {
	Text        string `storm:"id"` // primary key
	Translation string
	ErrorCode   translationError `storm:"index"`
	ErrorText   string
	Timestamp   int64
}

func NewCache(filePath string, cacheSize int) (*Cache, error) {
	db, err := sql.Open("sqlite3", filePath+"?_synchronous=1&_auto_vacuum=2&_journal_mode=WAL")
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	if _, err := db.Exec(fmt.Sprintf("PRAGMA cache_size = -%d", cacheSize)); err != nil {
		log.Errorf("Failed to increase page cache size! %s", err)
	} else {
		log.Infof("Page cache set to %d kib", cacheSize)
	}

	cache := &Cache{db: db}

	cache.migrateDatabase()

	return cache, nil
}

func (c *Cache) Close() error {
	log.Println("Writing checkpoint")
	c.db.Exec("PRAGMA wal_checkpoint")

	return c.db.Close()
}

func (c *Cache) Put(bucketName, text, translation string, cerr error) error {
	errorCode := errorNone
	errorText := ""

	if cerr != nil {
		errorText = cerr.Error()

		switch cerr.(type) {
		case translator.BadTranslationError:
			errorCode = errorBadTranslation
		default:
			errorCode = errorMinor
		}
	}

	stmt, err := c.db.Prepare("INSERT OR REPLACE INTO Translations(text, service, translation, errorCode, errorText, time) VALUES (?, ?, ?, ?, ?, ?)")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	_, err = stmt.Exec(text, bucketName, translation, errorCode, errorText, time.Now().UTC().Unix())
	if err != nil {
		log.Fatal(err)
	}

	return err
}

func (c *Cache) Get(bucketName, text string) (string, bool, error) {
	stmt, err := c.db.Prepare("SELECT id, translation, errorCode, errorText, time FROM Translations WHERE service = ? AND text = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	var found bool
	var id int64
	var translation string
	var errorCode translationError
	var errorText string
	var timestamp int64

	err = stmt.QueryRow(bucketName, text).Scan(&id, &translation, &errorCode, &errorText, &timestamp)
	if err != nil {
		return translation, found, nil
	}

	if translation != "" {
		found = true
	}

	if errorCode != errorNone {
		errorTime := time.Unix(timestamp, 0)

		if time.Since(errorTime) > getCacheExpiration(errorCode) {
			_, err = c.db.Exec("DELETE FROM Translations WHERE id = %s", id)
			if err != nil {
				log.Warn("unable to delete item:", err)
			}

			// Act as if nothing was found
			return "", false, nil
		}

		return "", found, fmt.Errorf("%s", errorText)
	}

	return translation, found, nil
}
<commit_msg>Avoid database is busy errors<commit_after>package cache

import (
	"database/sql"
	"fmt"
	"time"

	"gitgud.io/softashell/comfy-translator/translator"
	_ "github.com/mattn/go-sqlite3" // Sql driver
	log "github.com/sirupsen/logrus"
)

type translationError int

const (
	errorNone           translationError = iota // Everything is fine
	errorMinor                                  // Connection timed out or something like that
	errorBadTranslation                         // Returned really bad translation
)

const (
	cleanupInterval = (24 * time.Hour) * 7
)

type Cache struct {
	db *sql.DB
}

type Item struct {
	Text        string `storm:"id"` // primary key
	Translation string
	ErrorCode   translationError `storm:"index"`
	ErrorText   string
	Timestamp   int64
}

func NewCache(filePath string, cacheSize int) (*Cache, error) {
	db, err := sql.Open("sqlite3", filePath+"?cache=shared&mode=rwc&_synchronous=1&_auto_vacuum=2&_journal_mode=WAL")
	if err != nil {
		log.Fatal(err)
		return nil, err
	}

	if _, err := db.Exec(fmt.Sprintf("PRAGMA cache_size = -%d", cacheSize)); err != nil {
		log.Errorf("Failed to increase page cache size! %s", err)
	} else {
		log.Infof("Page cache set to %d kib", cacheSize)
	}

	cache := &Cache{db: db}

	cache.migrateDatabase()

	return cache, nil
}

func (c *Cache) Close() error {
	log.Println("Writing checkpoint")
	c.db.Exec("PRAGMA wal_checkpoint")

	return c.db.Close()
}

func (c *Cache) Put(bucketName, text, translation string, cerr error) error {
	errorCode := errorNone
	errorText := ""

	if cerr != nil {
		errorText = cerr.Error()

		switch cerr.(type) {
		case translator.BadTranslationError:
			errorCode = errorBadTranslation
		default:
			errorCode = errorMinor
		}
	}

	stmt, err := c.db.Prepare("INSERT OR REPLACE INTO Translations(text, service, translation, errorCode, errorText, time) VALUES (?, ?, ?, ?, ?, ?)")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	_, err = stmt.Exec(text, bucketName, translation, errorCode, errorText, time.Now().UTC().Unix())
	if err != nil {
		log.Fatal(err)
	}

	return err
}

func (c *Cache) Get(bucketName, text string) (string, bool, error) {
	stmt, err := c.db.Prepare("SELECT id, translation, errorCode, errorText, time FROM Translations WHERE service = ? AND text = ?")
	if err != nil {
		log.Fatal(err)
	}
	defer stmt.Close()

	var found bool
	var id int64
	var translation string
	var errorCode translationError
	var errorText string
	var timestamp int64

	err = stmt.QueryRow(bucketName, text).Scan(&id, &translation, &errorCode, &errorText, &timestamp)
	if err != nil {
		return translation, found, nil
	}

	if translation != "" {
		found = true
	}

	if errorCode != errorNone {
		errorTime := time.Unix(timestamp, 0)

		if time.Since(errorTime) > getCacheExpiration(errorCode) {
			_, err = c.db.Exec("DELETE FROM Translations WHERE id = %s", id)
			if err != nil {
				log.Warn("unable to delete item:", err)
			}

			// Act as if nothing was found
			return "", false, nil
		}

		return "", found, fmt.Errorf("%s", errorText)
	}

	return translation, found, nil
}

{'fraction_non_alphanumeric': 0.1303780964797914, 'fraction_numerical': 0.0045632333767926985, 'mean_word_length': 3.495970695970696, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 5, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4066927', 'n_tokens_mistral': 2216, 'n_tokens_neox': 2068, 'n_words': 580}