<commit_before>#include <stdio.h>
#include <assert.h>
#include <string.h>

#include <google/dense_hash_set>

#include <locale>
#include <list>
#include <iostream>
#include <string>

#include <re2/re2.h>

#include "smart_git.h"
#include "timer.h"

using google::dense_hash_set;
using re2::RE2;
using re2::StringPiece;
using namespace std;

#define CHUNK_SIZE (1 << 20)

class chunk_allocator {
public:
    chunk_allocator() : current_() {
        new_chunk();
    }

    char *alloc(size_t len) {
        assert(len < CHUNK_SIZE);
        if (alloc_ + len > end_)
            new_chunk();
        char *out = alloc_;
        alloc_ += len;
        return out;
    }

    list<StringPiece>::iterator begin () {
        return chunks_.begin();
    }

    list<StringPiece>::iterator end () {
        return chunks_.end();
    }

protected:
    void new_chunk() {
        if (current_ != 0)
            chunks_.push_back(StringPiece(current_, alloc_ - current_));
        current_ = new char[CHUNK_SIZE];
        alloc_ = current_;
        end_ = current_ + CHUNK_SIZE;
    }

    list<StringPiece> chunks_;
    char *current_;
    char *alloc_;
    char *end_;
};


/*
 * We special-case data() == NULL to provide an "empty" element for
 * dense_hash_set.
 *
 * StringPiece::operator== will consider a zero-length string equal to a
 * zero-length string with a NULL data().
 */
struct eqstr {
    bool operator()(const StringPiece& lhs, const StringPiece& rhs) const {
        if (lhs.data() == NULL && rhs.data() == NULL)
            return true;
        if (lhs.data() == NULL || rhs.data() == NULL)
            return false;
        return lhs == rhs;
    }
};

struct hashstr {
    locale loc;
    size_t operator()(const StringPiece &str) const {
        const collate<char> &coll = use_facet<collate<char> >(loc);
        return coll.hash(str.data(), str.data() + str.size());
    }
};

const StringPiece empty_string(NULL, 0);

typedef dense_hash_set<StringPiece, hashstr, eqstr> string_hash;

class code_counter {
public:
    code_counter(git_repository *repo)
        : repo_(repo), stats_()
    {
        lines_.set_empty_key(empty_string);
    }

    void walk_ref(const char *ref) {
        smart_object<git_commit> commit;
        smart_object<git_tree> tree;
        resolve_ref(commit, ref);
        git_commit_tree(tree, commit);

        walk_tree(tree);
    }

    void dump_stats() {
        printf("Bytes: %ld (dedup: %ld)\n", stats_.bytes, stats_.dedup_bytes);
        printf("Lines: %ld (dedup: %ld)\n", stats_.lines, stats_.dedup_lines);
    }

    bool match(RE2& pat) {
        list<StringPiece>::iterator it;
        StringPiece match;
        int matches = 0;

        for (it = alloc_.begin(); it != alloc_.end(); it++) {
            int pos = 0;
            while (pos < (*it).size()) {
                    if (!pat.Match(*it, pos, (*it).size(), RE2::UNANCHORED, &match, 1))
                        break;
                    assert(memchr(match.data(), '\n', match.size()) == NULL);
                    StringPiece line = find_line(*it, match);
                    printf("%.*s\n", line.size(), line.data());
                    pos = line.size() + line.data() - (*it).data();
                    if (++matches == 10)
                        return true;
                }
        }
        return matches > 0;
    }
protected:
    StringPiece find_line(const StringPiece& chunk, const StringPiece& match) {
        const char *start, *end;<commit_msg>Actually include the most recently-added chunk in the search.<commit_after>#include <stdio.h>
#include <assert.h>
#include <string.h>

#include <google/dense_hash_set>

#include <locale>
#include <list>
#include <iostream>
#include <string>

#include <re2/re2.h>

#include "smart_git.h"
#include "timer.h"

using google::dense_hash_set;
using re2::RE2;
using re2::StringPiece;
using namespace std;

#define CHUNK_SIZE (1 << 20)

class chunk_allocator {
public:
    chunk_allocator() : current_() {
        new_chunk();
    }

    char *alloc(size_t len) {
        assert(len < CHUNK_SIZE);
        if (alloc_ + len > end_)
            new_chunk();
        char *out = alloc_;
        alloc_ += len;
        return out;
    }

    list<StringPiece>::iterator begin () {
        return chunks_.begin();
    }

    list<StringPiece>::iterator end () {
        return chunks_.end();
    }

    void finish(void) {
        new_chunk();
    }

protected:
    void new_chunk() {
        if (current_ != 0)
            chunks_.push_back(StringPiece(current_, alloc_ - current_));
        current_ = new char[CHUNK_SIZE];
        alloc_ = current_;
        end_ = current_ + CHUNK_SIZE;
    }

    list<StringPiece> chunks_;
    char *current_;
    char *alloc_;
    char *end_;
};


/*
 * We special-case data() == NULL to provide an "empty" element for
 * dense_hash_set.
 *
 * StringPiece::operator== will consider a zero-length string equal to a
 * zero-length string with a NULL data().
 */
struct eqstr {
    bool operator()(const StringPiece& lhs, const StringPiece& rhs) const {
        if (lhs.data() == NULL && rhs.data() == NULL)
            return true;
        if (lhs.data() == NULL || rhs.data() == NULL)
            return false;
        return lhs == rhs;
    }
};

struct hashstr {
    locale loc;
    size_t operator()(const StringPiece &str) const {
        const collate<char> &coll = use_facet<collate<char> >(loc);
        return coll.hash(str.data(), str.data() + str.size());
    }
};

const StringPiece empty_string(NULL, 0);

typedef dense_hash_set<StringPiece, hashstr, eqstr> string_hash;

class code_counter {
public:
    code_counter(git_repository *repo)
        : repo_(repo), stats_()
    {
        lines_.set_empty_key(empty_string);
    }

    void walk_ref(const char *ref) {
        smart_object<git_commit> commit;
        smart_object<git_tree> tree;
        resolve_ref(commit, ref);
        git_commit_tree(tree, commit);

        walk_tree(tree);
    }

    void dump_stats() {
        printf("Bytes: %ld (dedup: %ld)\n", stats_.bytes, stats_.dedup_bytes);
        printf("Lines: %ld (dedup: %ld)\n", stats_.lines, stats_.dedup_lines);
    }

    bool match(RE2& pat) {
        list<StringPiece>::iterator it;
        StringPiece match;
        int matches = 0;
        alloc_.finish();

        for (it = alloc_.begin(); it != alloc_.end(); it++) {
            int pos = 0;
            while (pos < (*it).size()) {
                    if (!pat.Match(*it, pos, (*it).size(), RE2::UNANCHORED, &match, 1))
                        break;
                    assert(memchr(match.data(), '\n', match.size()) == NULL);
                    StringPiece line = find_line(*it, match);
                    printf("%.*s\n", line.size(), line.data());
                    pos = line.size() + line.data() - (*it).data();
                    if (++matches == 10)
                        return true;
                }
        }
        return matches > 0;
    }
protected:
    StringPiece find_line(const StringPiece& chunk, const StringPiece& match) {
        const char *start, *end;
{'fraction_non_alphanumeric': 0.13991246646901032, 'fraction_numerical': 0.005082592121982211, 'mean_word_length': 2.2630124366651314, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 45, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6458423', 'n_tokens_mistral': 2298, 'n_tokens_neox': 2238, 'n_words': 553}