<commit_before>package main

import (
    "flag"
    "log"
    "os"
    "math/big"
    "time"

    "github.com/tooru/pnbot/prime"
    "github.com/dghubble/go-twitter/twitter"
    "github.com/dghubble/oauth1"
)

func main() {
    log.SetFlags(log.Ldate | log.Ltime | log.Llongfile)

    flags := flag.NewFlagSet("pnbot", flag.ExitOnError)
    consumerKey := flags.String("ck", "", "Twitter Consumer Key")
    consumerSecret := flags.String("cs", "", "Twitter Consumer Secret")
    accessToken := flags.String("at", "", "Twitter Access Token")
    accessSecret := flags.String("as", "", "Twitter Access Secret")
    flags.Parse(os.Args[1:])

    if *consumerKey == "" || *consumerSecret == "" || *accessToken == "" || *accessSecret == "" {
        log.Fatal("Consumer key/secret and Access token/secret required")
    }

    config := oauth1.NewConfig(*consumerKey, *consumerSecret)
    token := oauth1.NewToken(*accessToken, *accessSecret)

    httpClient := config.Client(oauth1.NoContext, token)
    client := twitter.NewClient(httpClient)

    
    primes := prime.NewPrime()

    pnTweet(client, primes)
}

func pnTweet(client *twitter.Client, primes *prime.Prime) {
    maxPrime, err := getMaxPrime(client)
    if err != nil {
        log.Fatalf("getMaxPrime: %v\n", err)
        return
    }

    ch := make(chan *big.Int, 1000)

    go makePrimes(primes, maxPrime, ch)
    tweetPrimes(client, ch)
}

func getMaxPrime(client *twitter.Client) (prime *big.Int, err error) {
    tweets, _, err := client.Timelines.HomeTimeline(&twitter.HomeTimelineParams{
        SinceID: 0,
    })
    if err != nil {
        return nil, err
    }

    var maxPrime big.Int

    for _, tweet := range tweets {
        if _, ok := maxPrime.SetString(tweet.Text, 10); ok {
            log.Printf("continue from '%s'", tweet.Text)
            return &maxPrime, nil
        }
        log.Printf("skip '%v'", tweet.Text)
    }
    maxPrime.SetInt64(0)
    return &maxPrime, nil
}

func makePrimes(primes *prime.Prime, maxPrime *big.Int, ch chan *big.Int) {
    var prime *big.Int
    var err error

    for {
        prime, err = primes.Next()
        if err != nil {
            log.Fatalf("makePrimes: %v\n", err)
            ch <- nil
            return
        }            

        if prime.Cmp(maxPrime) > 0 {
            break;
        }
    }

    for {
        ch <- prime

        prime, err = primes.Next()
        if err != nil {
            log.Fatalf("makePrimes: %v\n", err)
            ch <- nil
            return
        }            
    }
}

func tweetPrimes(client *twitter.Client, ch chan *big.Int) {
    for {
        prime := <- ch
        if prime == nil {
            log.Fatalf("prime generator was died")
            return
        }

        text := prime.Text(10)
        _, _, err := client.Statuses.Update(text, nil)
        if err != nil {
            log.Fatalf("Tweet error: %v\n", err)
            return
        }
        log.Printf("tweet %s\n", text)
        time.Sleep(30 * time.Second)
    }
}
<commit_msg>Modified pnbot.go.<commit_after>package main

import (
    "flag"
    "log"
    "os"
    "math/big"
    "time"

    "github.com/tooru/pnbot/prime"
    "github.com/dghubble/go-twitter/twitter"
    "github.com/dghubble/oauth1"
)

const (
    queueSize = 100000
    maxRetry = 10
)

func main() {
    log.SetFlags(log.Ldate | log.Ltime | log.Llongfile)

    flags := flag.NewFlagSet("pnbot", flag.ExitOnError)
    consumerKey := flags.String("ck", "", "Twitter Consumer Key")
    consumerSecret := flags.String("cs", "", "Twitter Consumer Secret")
    accessToken := flags.String("at", "", "Twitter Access Token")
    accessSecret := flags.String("as", "", "Twitter Access Secret")
    flags.Parse(os.Args[1:])

    if *consumerKey == "" || *consumerSecret == "" || *accessToken == "" || *accessSecret == "" {
        log.Fatal("Consumer key/secret and Access token/secret required")
    }

    config := oauth1.NewConfig(*consumerKey, *consumerSecret)
    token := oauth1.NewToken(*accessToken, *accessSecret)

    httpClient := config.Client(oauth1.NoContext, token)
    client := twitter.NewClient(httpClient)

    
    primes := prime.NewPrime()

    pnTweet(client, primes)
}

func pnTweet(client *twitter.Client, primes *prime.Prime) {
    maxPrime, err := getMaxPrime(client)
    if err != nil {
        log.Fatalf("getMaxPrime: %v\n", err)
        return
    }

    ch := make(chan *big.Int, queueSize)

    go makePrimes(primes, maxPrime, ch)
    tweetPrimes(client, ch)
}

func getMaxPrime(client *twitter.Client) (prime *big.Int, err error) {
    tweets, _, err := client.Timelines.HomeTimeline(&twitter.HomeTimelineParams{
        SinceID: 0,
    })
    if err != nil {
        return nil, err
    }

    var maxPrime big.Int

    for _, tweet := range tweets {
        if _, ok := maxPrime.SetString(tweet.Text, 10); ok {
            log.Printf("continue from '%s'", tweet.Text)
            return &maxPrime, nil
        }
        log.Printf("skip '%v'", tweet.Text)
    }
    maxPrime.SetInt64(0)
    return &maxPrime, nil
}

func makePrimes(primes *prime.Prime, maxPrime *big.Int, ch chan *big.Int) {
    var prime *big.Int
    var err error

    for {
        prime, err = primes.Next()
        if err != nil {
            log.Fatalf("makePrimes: %v\n", err)
            ch <- nil
            return
        }            

        if prime.Cmp(maxPrime) > 0 {
            break;
        }
        log.Printf("makePrimes: skip %v\n", prime)
    }

    for {
        ch <- prime

        prime, err = primes.Next()
        if err != nil {
            log.Fatalf("makePrimes: %v\n", err)
            ch <- nil
            return
        }            
    }
}

func tweetPrimes(client *twitter.Client, ch chan *big.Int) {
    for {
        prime := <- ch
        if prime == nil {
            log.Fatalf("prime generator was died")
            return
        }

        text := prime.Text(10)
        retry := 0
        for {
            _, _, err := client.Statuses.Update(text, nil)
            if err == nil {
                break
            }
            if retry >= maxRetry {
                log.Fatalf("Too many tweet error: %v\n", err)
                return
            }
            log.Printf("Tweet error: %v\n", err)
            time.Sleep(10 * time.Minute)
            continue
        }
        log.Printf("tweet %s\n", text)
        time.Sleep(1 * time.Minute)
    }
}

{'fraction_non_alphanumeric': 0.13990040460628697, 'fraction_numerical': 0.0071584189231248055, 'mean_word_length': 1.979601298099212, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 6, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1802833', 'n_tokens_mistral': 2180, 'n_tokens_neox': 2054, 'n_words': 458}