<commit_before>#include <microscopes/lda/model.hpp>
#include <microscopes/lda/data.hpp>
#include <microscopes/lda/random_docs.hpp>
#include <microscopes/common/macros.hpp>
#include <microscopes/common/recarray/dataview.hpp>
#include <microscopes/models/distributions.hpp>
#include <microscopes/common/random_fwd.hpp>

#include <random>
#include <iostream>
#include <time.h>

using namespace std;
using namespace distributions;
using namespace microscopes;
using namespace microscopes::common;
using namespace microscopes::common::recarray;


static void
test_small()
{
    rng_t r(time(NULL)); //

    std::cout << "creating model definition...";
    lda::model_definition def(3, 4);
    std::vector<std::vector<size_t>> docs = {{0, 1, 2}, {1, 2, 3}, {3, 3}};
    lda::state state(def, .1, .5, .1, docs, r);
    std::cout << " complete" << std::endl;

    for(unsigned i = 0; i < 100; ++i){<commit_msg>Auto count words in docs<commit_after>#include <microscopes/lda/model.hpp>
#include <microscopes/lda/data.hpp>
#include <microscopes/lda/random_docs.hpp>
#include <microscopes/common/macros.hpp>
#include <microscopes/common/recarray/dataview.hpp>
#include <microscopes/models/distributions.hpp>
#include <microscopes/common/random_fwd.hpp>

#include <random>
#include <iostream>
#include <time.h>

using namespace std;
using namespace distributions;
using namespace microscopes;
using namespace microscopes::common;
using namespace microscopes::common::recarray;

size_t
num_unique_words_in_docs(const std::vector< std::vector<size_t> > &docs){
    std::set<size_t> words;
    for(auto doc: docs)
        for(auto word: doc)
            words.insert(word);
    return words.size();
}



static void
test_small()
{
    rng_t r(time(NULL)); //

    std::cout << "creating model definition...";
    std::vector<std::vector<size_t>> docs = {{0, 1}, {2, 3}};
    lda::model_definition def(docs.size(), num_unique_words_in_docs(docs));
    lda::state state(def, .5, .01, .5, docs, r);
    std::cout << " complete" << std::endl;

    for(unsigned i = 0; i < 100; ++i){
{'fraction_non_alphanumeric': 0.16141541444498303, 'fraction_numerical': 0.014057198254968492, 'mean_word_length': 4.931034482758621, 'pattern_counts': {'":': 0, '<': 44, '<?xml version=': 0, '>': 30, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2937960', 'n_tokens_mistral': 774, 'n_tokens_neox': 723, 'n_words': 161}