<commit_before>/*
Command-line client that uses the janky ebooker package to generate tweets
easily. See --help for all the settings.
*/
package main

import (
    "ebooker"

    "flag"
    "fmt"
    "math/rand"
    "time"
)

func main() {

    // Parse flags, initialize data...
    var username string
    var prefixLen, numTweets int
    var silent, debug, reps, timestamps bool

    flag.StringVar(&username, "user", "laurelita", "Twitter user to base output off of.")
    flag.IntVar(&prefixLen, "prefixLength", 1, "length of prefix")
    flag.IntVar(&numTweets, "numTweets", 50, "the number of tweets to produce")
    flag.BoolVar(&silent, "silent", true, "Generate only the tweets, without other status information.")
    flag.BoolVar(&reps, "representations", false, "Treat various representations (e.g. \"Its/it's/IT'S\") as the same in generation.")
    flag.BoolVar(&debug, "debug", false, "Print debugging information.")
    flag.BoolVar(&timestamps, "timestamps", false, "Print log/debug with timestamps.")


    flag.Parse()
    rand.Seed(time.Now().UnixNano())

    logger := ebooker.GetLogMaster(silent, debug, timestamps)
    dh := ebooker.GetDataHandle("./ebooker_tweets.db", &logger)
    defer dh.Cleanup()


    // get tweets from persistent storage
    logger.StatusWrite("Reading from persistent storage...\n")
    oldTweets := dh.GetTweetsFromStorage(username)

    tf := ebooker.GetTweetFetcher(&logger)

    var newTweets []ebooker.TweetData
    if len(oldTweets) == 0 {
        logger.StatusWrite("Found no tweets for %s, doing a deep dive to retrieve their history.\n", username)
        newTweets = tf.DeepDive(username)
    } else {
        logger.StatusWrite("Found %d tweets for %s.\n", len(oldTweets), username)
        newest := oldTweets[len(oldTweets) - 1]
        newTweets = tf.GetTimelineFromRequest(username, &newest)
    }

    // update the persistent storage
    logger.StatusWrite("Inserting %d new tweets into persistent storage.\n", len(newTweets))
    dh.InsertFreshTweets(username, newTweets)

    // fetch or create a Generator
    gen := ebooker.CreateGenerator(prefixLen, 140, &logger)
    if reps {
        gen.CanonicalizeSources()
    }

    // Seed the Generator
    for _, tweet := range oldTweets {
        gen.AddSeeds(tweet.Text)
    }
    for _, tweet := range newTweets {
        gen.AddSeeds(tweet.Text)
    }

    // Generate some faux tweets. Print them!
    logger.StatusWrite("Outputting nonsense tweets for %s:\n", username)
    var format string
    if silent {
        format = "%s\n"
    } else {
        format = "  %s\n"
    }

    for i := 0; i < numTweets; i++ {
        fmt.Printf(format, gen.GenerateText())
    }
}
<commit_msg>Support for multiple user inputs on command-line.<commit_after>/*
Command-line client that uses the janky ebooker package to generate tweets
easily. See --help for all the settings.
*/
package main

import (
    "ebooker"

    "flag"
    "fmt"
    "math/rand"
    "strings"
    "time"
)

func main() {

    // Parse flags, initialize data...
    var userlist string
    var prefixLen, numTweets int
    var silent, debug, reps, timestamps bool

    flag.StringVar(&userlist, "users", "laurelita", "Comma-separated list of twitter users to base output off of.")
    flag.IntVar(&prefixLen, "prefixLength", 1, "length of prefix")
    flag.IntVar(&numTweets, "numTweets", 50, "the number of tweets to produce")
    flag.BoolVar(&silent, "silent", true, "Generate only the tweets, without other status information.")
    flag.BoolVar(&reps, "representations", false, "Treat various representations (e.g. \"Its/it's/IT'S\") as the same in generation.")
    flag.BoolVar(&debug, "debug", false, "Print debugging information.")
    flag.BoolVar(&timestamps, "timestamps", false, "Print log/debug with timestamps.")


    flag.Parse()
    rand.Seed(time.Now().UnixNano())

    logger := ebooker.GetLogMaster(silent, debug, timestamps)
    dh := ebooker.GetDataHandle("./ebooker_tweets.db", &logger)
    defer dh.Cleanup()

    users := strings.Split(userlist, ",")
    var sourcestrings[]string

    for _, username := range users {
        // get tweets from persistent storage
        logger.StatusWrite("Reading from persistent storage for %s...\n", username)
        oldTweets := dh.GetTweetsFromStorage(username)

        tf := ebooker.GetTweetFetcher(&logger)

        var newTweets ebooker.Tweets
        if len(oldTweets) == 0 {
            logger.StatusWrite("Found no tweets for %s, doing a deep dive to retrieve their history.\n", username)
            newTweets = tf.DeepDive(username)
        } else {
            logger.StatusWrite("Found %d tweets for %s.\n", len(oldTweets), username)
            newest := oldTweets[len(oldTweets) - 1]
            newTweets = tf.GetTimelineFromRequest(username, &newest)
        }

        // update the persistent storage
        logger.StatusWrite("Inserting %d new tweets into persistent storage.\n", len(newTweets))
        dh.InsertFreshTweets(username, newTweets)

		copyFrom(&sourcestrings, &oldTweets)
		copyFrom(&sourcestrings, &newTweets)
    }

	// fetch or create a Generator
	gen := ebooker.CreateGenerator(prefixLen, 140, &logger)
	if reps {
		gen.CanonicalizeSources()
	}

	// Seed the Generator
	for _, str := range sourcestrings {
		gen.AddSeeds(str)
	}

    // Generate some faux tweets. Print them!
    logger.StatusWrite("Outputting nonsense tweets for \"%v\":\n", userlist)
    var format string
    if silent {
        format = "%s\n"
    } else {
        format = "  %s\n"
    }

    for i := 0; i < numTweets; i++ {
        fmt.Printf(format, gen.GenerateText())
    }
}


func copyFrom(dst *[]string, src *ebooker.Tweets) {
	for _, str := range *src {
		*dst = append(*dst, str.Text)
	}
}
{'fraction_non_alphanumeric': 0.12606355270012154, 'fraction_numerical': 0.0031255426289286334, 'mean_word_length': 3.01673640167364, 'pattern_counts': {'":': 1, '<': 5, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3323567', 'n_tokens_mistral': 1931, 'n_tokens_neox': 1822, 'n_words': 506}