<commit_before>package main

import (
	"github.com/octokit/go-octokit/octokit"
)

const (
	importBatchSize = 200
)

func importWorker(impq chan *octokit.User) {
	for {
		users := []*octokit.User{}
		for i := 0; i < importBatchSize; i++ {
			users = append(users, <-impq)
		}
		importUsers(users)
	}
}

func importUser(user *octokit.User) {
	_, err := db.Exec(`
		UPDATE users
		SET users.login = ?, users.avatar_url = ?, users.public_repos = ?
		WHERE id = ?;
		`,
		user.Login,
		user.AvatarURL,
		user.PublicRepos,
		user.ID,
	)
	logError(err)
}

func importUsers(users []*octokit.User) {
	if len(users) == 0 {
		return
	}

	sql := "INSERT INTO users (id, login, avatar_url, public_repos) VALUES "
	values := []interface{}{}
	for _, user := range users {
		sql += "(?,?,?,?),"
		values = append(values, user.ID, user.Login, user.AvatarURL, user.PublicRepos)
	}
	sql = sql[0 : len(sql)-1] // trim last ,
	sql += `
		ON DUPLICATE KEY UPDATE
		id=VALUES(id),
		login=VALUES(login),
		avatar_url=VALUES(avatar_url),
		public_repos=VALUES(public_repos);
	`

	_, err := db.Exec(sql, values...)
	logError(err)
}<commit_msg>created_at and updated_at can't be null<commit_after>package main

import (
	"github.com/octokit/go-octokit/octokit"
)

const (
	importBatchSize = 200
)

func importWorker(impq chan *octokit.User) {
	for {
		users := []*octokit.User{}
		for i := 0; i < importBatchSize; i++ {
			users = append(users, <-impq)
		}
		importUsers(users)
	}
}

func importUser(user *octokit.User) {
	_, err := db.Exec(`
		UPDATE users
		SET users.login = ?, users.avatar_url = ?, users.public_repos = ?
		WHERE id = ?;
		`,
		user.Login,
		user.AvatarURL,
		user.PublicRepos,
		user.ID,
	)
	logError(err)
}

func importUsers(users []*octokit.User) {
	if len(users) == 0 {
		return
	}

	sql := `
		INSERT INTO users (
			id,
			login,
			avatar_url,
			public_repos,
			created_at,
			updated_at
		) VALUES
	`
	values := []interface{}{}
	for _, user := range users {
		sql += "(?,?,?,?,?,?),"
		values = append(
			values,
			user.ID,
			user.Login,
			user.AvatarURL,
			user.PublicRepos,
			user.CreatedAt,
			user.UpdatedAt,
		)
	}
	sql = sql[0 : len(sql)-1] // trim last ,
	sql += `
		ON DUPLICATE KEY UPDATE
		login=VALUES(login),
		avatar_url=VALUES(avatar_url),
		public_repos=VALUES(public_repos),
		created_at=VALUES(created_at),
		updated_at=VALUES(updated_at);
	`

	_, err := db.Exec(sql, values...)
	logError(err)
}
{'fraction_non_alphanumeric': 0.16343375980189848, 'fraction_numerical': 0.005777961205117622, 'mean_word_length': 3.4723247232472323, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2864329', 'n_tokens_mistral': 1088, 'n_tokens_neox': 974, 'n_words': 189}