<commit_before>package com.github.k0kubun.gitstar_ranking.worker

import com.github.k0kubun.gitstar_ranking.config.Config
import java.util.concurrent.BlockingQueue
import org.skife.jdbi.v2.DBI
import com.github.k0kubun.gitstar_ranking.github.GitHubClientBuilder
import kotlin.Throws
import java.lang.Exception
import java.util.concurrent.TimeUnit
import com.github.k0kubun.gitstar_ranking.github.GitHubClient
import com.github.k0kubun.gitstar_ranking.model.User
import com.github.k0kubun.gitstar_ranking.repository.dao.LastUpdateDao
import com.github.k0kubun.gitstar_ranking.repository.dao.UserDao
import org.skife.jdbi.v2.TransactionStatus
import java.io.IOException
import java.lang.InterruptedException
import java.sql.Timestamp
import java.time.Instant
import java.time.temporal.ChronoUnit
import java.util.ArrayList
import org.skife.jdbi.v2.Handle
import org.slf4j.LoggerFactory

// Users with too many repositories. To be fixed later.
val PENDING_USERS = listOf(
    "GITenberg",
    "gitpan",
    "the-domains",
    "wp-plugins",
)

// Scan all starred users
class UserStarScanWorker(config: Config) : UpdateUserWorker(config.databaseConfig.dataSource) {
    private val userStarScanQueue: BlockingQueue<Boolean> = config.queueConfig.userStarScanQueue
    override val dbi: DBI = DBI(config.databaseConfig.dataSource)
    override val clientBuilder: GitHubClientBuilder = GitHubClientBuilder(config.databaseConfig.dataSource)
    private val updateThreshold: Timestamp = Timestamp.from(Instant.now().minus(THRESHOLD_DAYS, ChronoUnit.DAYS))

    @Throws(Exception::class)
    override fun perform() {
        while (userStarScanQueue.poll(5, TimeUnit.SECONDS) == null) {
            if (isStopped) {
                return
            }
        }
        val client = clientBuilder.buildForUser(TOKEN_USER_ID)
        LOG.info(String.format("----- started UserStarScanWorker (API: %s/5000) -----", client.rateLimitRemaining))
        dbi.open().use { handle ->
            var numUsers = 1000 // 2 * (1000 / 30 min) ≒ 4000 / hour
            var numChecks = 2000 // Avoid issuing too many queries by skips
            while (numUsers > 0 && numChecks > 0 && !isStopped) {
                // Find a current cursor
                var lastUpdatedId = handle.attach(LastUpdateDao::class.java).getCursor(LastUpdateDao.STAR_SCAN_USER_ID)
                var stars = handle.attach(LastUpdateDao::class.java).getCursor(LastUpdateDao.STAR_SCAN_STARS)
                if (stars == 0L) {
                    stars = handle.attach(UserDao::class.java).maxStargazersCount()
                }

                // Query a next batch
                var users: List<User> = ArrayList()<commit_msg>Skip gitter-badger<commit_after>package com.github.k0kubun.gitstar_ranking.worker

import com.github.k0kubun.gitstar_ranking.config.Config
import java.util.concurrent.BlockingQueue
import org.skife.jdbi.v2.DBI
import com.github.k0kubun.gitstar_ranking.github.GitHubClientBuilder
import kotlin.Throws
import java.lang.Exception
import java.util.concurrent.TimeUnit
import com.github.k0kubun.gitstar_ranking.github.GitHubClient
import com.github.k0kubun.gitstar_ranking.model.User
import com.github.k0kubun.gitstar_ranking.repository.dao.LastUpdateDao
import com.github.k0kubun.gitstar_ranking.repository.dao.UserDao
import org.skife.jdbi.v2.TransactionStatus
import java.io.IOException
import java.lang.InterruptedException
import java.sql.Timestamp
import java.time.Instant
import java.time.temporal.ChronoUnit
import java.util.ArrayList
import org.skife.jdbi.v2.Handle
import org.slf4j.LoggerFactory

// Users with too many repositories. To be fixed later.
val PENDING_USERS = listOf(
    "GITenberg",
    "gitpan",
    "the-domains",
    "wp-plugins",
    "gitter-badger",
)

// Scan all starred users
class UserStarScanWorker(config: Config) : UpdateUserWorker(config.databaseConfig.dataSource) {
    private val userStarScanQueue: BlockingQueue<Boolean> = config.queueConfig.userStarScanQueue
    override val dbi: DBI = DBI(config.databaseConfig.dataSource)
    override val clientBuilder: GitHubClientBuilder = GitHubClientBuilder(config.databaseConfig.dataSource)
    private val updateThreshold: Timestamp = Timestamp.from(Instant.now().minus(THRESHOLD_DAYS, ChronoUnit.DAYS))

    @Throws(Exception::class)
    override fun perform() {
        while (userStarScanQueue.poll(5, TimeUnit.SECONDS) == null) {
            if (isStopped) {
                return
            }
        }
        val client = clientBuilder.buildForUser(TOKEN_USER_ID)
        LOG.info(String.format("----- started UserStarScanWorker (API: %s/5000) -----", client.rateLimitRemaining))
        dbi.open().use { handle ->
            var numUsers = 1000 // 2 * (1000 / 30 min) ≒ 4000 / hour
            var numChecks = 2000 // Avoid issuing too many queries by skips
            while (numUsers > 0 && numChecks > 0 && !isStopped) {
                // Find a current cursor
                var lastUpdatedId = handle.attach(LastUpdateDao::class.java).getCursor(LastUpdateDao.STAR_SCAN_USER_ID)
                var stars = handle.attach(LastUpdateDao::class.java).getCursor(LastUpdateDao.STAR_SCAN_STARS)
                if (stars == 0L) {
                    stars = handle.attach(UserDao::class.java).maxStargazersCount()
                }

                // Query a next batch
                var users: List<User> = ArrayList()
{'fraction_non_alphanumeric': 0.09977786005183266, 'fraction_numerical': 0.014068863383931877, 'mean_word_length': 3.2983293556085918, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3440840', 'n_tokens_mistral': 1739, 'n_tokens_neox': 1636, 'n_words': 317}