<commit_before>package main

import (
	"fmt"
	"log"
	"strings"
)

const baseTable = `CREATE TABLE Words (
    Nick VARCHAR(32),
    %s
    primary KEY (Nick));
    `

func updateWords(nick, message string) error {
	for word, regex := range badWords {
		numwords := len(regex.FindAllString(strings.ToLower(message), -1))
		if numwords == 0 {
			continue
		}
		tx, err := db.Begin()
		if err != nil {
			return err
		}
		_, err = tx.Exec(`SELECT * FROM Words WHERE Nick=? FOR UPDATE`, nick)
		if err != nil {
			return err
		}
		_, err = tx.Exec(fmt.Sprintf(`INSERT INTO Words (Nick, %[1]s) VALUES (?, ?) ON DUPLICATE KEY UPDATE %[1]s=%[1]s+VALUES(%[1]s)`, word), nick, numwords)
		if err != nil {
			return err
		}
		err = tx.Commit()
		if err != nil {
			return err
		}
	}
	return nil
}

func genTables() {
	log.Println("Regenerating Words table")
	wordList := ""
	for _, word := range config.BadWords {
		wordList += word.Word + " INT(32) NOT NULL DEFAULT 0, "
	}

	_, err := db.Exec(`DROP TABLE IF EXISTS Words`)
	if err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec(fmt.Sprintf(baseTable, wordList))
	if err != nil {
		log.Fatal(err)
	}

	rows, err := db.Query(`SELECT Nick, Message from messages WHERE channel='#geekhack'`)
	if err != nil {
		log.Fatal(err)
	}
	for rows.Next() {
		var nick, message string<commit_msg>clean up updateWords to only use one transaction per row<commit_after>package main

import (
	"fmt"
	"log"
	"strings"
)

const baseTable = `CREATE TABLE Words (
    Nick VARCHAR(32),
    %s
    primary KEY (Nick));
    `

func updateWords(nick, message string) error {
	tx, err := db.Begin()
	if err != nil {
		return err
	}
	_, err = tx.Exec(`SELECT * FROM Words WHERE Nick=? FOR UPDATE`, nick)
	if err != nil {
		return err
	}
	for word, regex := range badWords {
		numwords := len(regex.FindAllString(strings.ToLower(message), -1))
		if numwords == 0 {
			continue
		}
		_, err = tx.Exec(fmt.Sprintf(`INSERT INTO Words (Nick, %[1]s) VALUES (?, ?)`+
			` ON DUPLICATE KEY UPDATE %[1]s=%[1]s+VALUES(%[1]s)`, word), nick, numwords)
		if err != nil {
			return err
		}
	}
	err = tx.Commit()
	if err != nil {
		return err
	}
	return nil
}

func genTables() {
	log.Println("Regenerating Words table")
	wordList := ""
	for _, word := range config.BadWords {
		wordList += word.Word + " INT(32) NOT NULL DEFAULT 0, "
	}

	_, err := db.Exec(`DROP TABLE IF EXISTS Words`)
	if err != nil {
		log.Fatal(err)
	}
	_, err = db.Exec(fmt.Sprintf(baseTable, wordList))
	if err != nil {
		log.Fatal(err)
	}

	rows, err := db.Query(`SELECT Nick, Message from messages WHERE channel='#geekhack'`)
	if err != nil {
		log.Fatal(err)
	}
	for rows.Next() {
		var nick, message string
{'fraction_non_alphanumeric': 0.15461997019374069, 'fraction_numerical': 0.00819672131147541, 'mean_word_length': 3.261904761904762, 'pattern_counts': {'":': 0, '<': 3, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6456805', 'n_tokens_mistral': 1091, 'n_tokens_neox': 987, 'n_words': 284}