<commit_before>#include <stdio.h>
#include <assert.h>
#include <string.h>

#include <google/dense_hash_set>

#include <locale>
#include <list>
#include <iostream>
#include <string>
#include <atomic>

#include <re2/re2.h>

#include "smart_git.h"
#include "timer.h"
#include "thread_queue.h"
#include "thread_pool.h"

using google::dense_hash_set;
using re2::RE2;
using re2::StringPiece;
using namespace std;

#define CHUNK_SIZE (1 << 20)
#define MAX_GAP    (1 << 12)
#define MAX_MATCHES 10

struct search_file {
    string path;
    const char *ref;
    git_oid oid;
};

struct chunk_file {
    search_file *file;
    int left;
    int right;
    void expand(int l, int r) {
        left  = min(left, l);
        right = max(right, r);
    }

    bool operator<(const chunk_file& rhs) const {
        return left < rhs.left;
    }
};

struct match_result {
    search_file *file;
    int lno;
    StringPiece line;
};

#define CHUNK_MAGIC 0xC407FADE

struct chunk {
    int size;
    unsigned magic;
    vector<chunk_file> files;
    vector<chunk_file> cur_file;
    char data[0];

    chunk()
        : size(0), magic(CHUNK_MAGIC), files() {
    }

    void add_chunk_file(search_file *sf, const StringPiece &line) {
        int l = line.data() - data;
        int r = l + line.size();
        chunk_file *f = NULL;
        int min_dist = numeric_limits<int>::max(), dist;
        for (vector<chunk_file>::iterator it = cur_file.begin();
             it != cur_file.end(); it ++) {
            if (l <= it->left)
                dist = max(0, it->left - r);
            else if (r >= it->right)
                dist = max(0, l - it->right);
            else
                dist = 0;
            assert(dist == 0 || r < it->left || l > it->right);
            if (dist < min_dist) {
                min_dist = dist;
                f = &(*it);
            }
        }
        if (f && min_dist < MAX_GAP) {
            f->expand(l, r);
            return;
        }
        cur_file.push_back(chunk_file());
        chunk_file &cf = cur_file.back();
        cf.file = sf;
        cf.left = l;
        cf.right = r;
    }

    void finish_file() {
        int right = -1;
        sort(cur_file.begin(), cur_file.end());
        for (vector<chunk_file>::iterator it = cur_file.begin();
             it != cur_file.end(); it ++) {
            assert(right < it->left);
            right = max(right, it->right);
        }
        files.insert(files.end(), cur_file.begin(), cur_file.end());
        cur_file.clear();
    }

    static chunk* from_str(const char *p) {
        chunk *out = reinterpret_cast<chunk*>
            (reinterpret_cast<uintptr_t>(p) & ~(CHUNK_SIZE - 1));
        assert(out->magic == CHUNK_MAGIC);
        return out;
    }
};

#define CHUNK_SPACE  (CHUNK_SIZE - (sizeof(chunk)))

chunk *alloc_chunk() {
    void *p;
    if (posix_memalign(&p, CHUNK_SIZE, CHUNK_SIZE) != 0)
        return NULL;
    return new(p) chunk;
};

class chunk_allocator {
public:
    chunk_allocator() : current_() {
        new_chunk();
    }

    char *alloc(size_t len) {
        assert(len < CHUNK_SPACE);
        if ((current_->size + len) > CHUNK_SPACE)
            new_chunk();
        char *out = current_->data + current_->size;
        current_->size += len;
        return out;
    }

    list<chunk*>::iterator begin () {
        return chunks_.begin();
    }

    typename list<chunk*>::iterator end () {
        return chunks_.end();
    }

    chunk *current_chunk() {
        return current_;
    }

protected:
    void new_chunk() {
        current_ = alloc_chunk();
        chunks_.push_back(current_);
    }

    list<chunk*> chunks_;
    chunk *current_;
};

/*
 * We special-case data() == NULL to provide an "empty" element for
 * dense_hash_set.
 *
 * StringPiece::operator== will consider a zero-length string equal to a
 * zero-length string with a NULL data().
 */
struct eqstr {
    bool operator()(const StringPiece& lhs, const StringPiece& rhs) const {
        if (lhs.data() == NULL && rhs.data() == NULL)
            return true;
        if (lhs.data() == NULL || rhs.data() == NULL)
            return false;
        return lhs == rhs;
    }
};

struct hashstr {
    locale loc;
    size_t operator()(const StringPiece &str) const {
        const collate<char> &coll = use_facet<collate<char> >(loc);
        return coll.hash(str.data(), str.data() + str.size());
    }
};

const StringPiece empty_string(NULL, 0);

typedef dense_hash_set<StringPiece, hashstr, eqstr> string_hash;

class code_counter;

class searcher {
public:
    searcher(code_counter *cc, thread_queue<match_result*> &queue, RE2& pat) :
        cc_(cc), pat_(pat), queue_(queue),
        matches_(0), searched_(0), hit_rate_(0) {
    }

    bool operator()(const chunk *chunk) {
        if (chunk == NULL) {
            queue_.push(NULL);
            return true;
        }
        StringPiece str(chunk->data, chunk->size);
        StringPiece match;
        int pos = 0;
        while (pos < str.size()) {
            if (!pat_.Match(str, pos, str.size(), RE2::UNANCHORED, &match, 1))
                break;
            assert(memchr(match.data(), '\n', match.size()) == NULL);
            StringPiece line = find_line(str, match);
            find_match(line);
            pos = line.size() + line.data() - str.data();
        }
        return false;
    }

    int approx_matches() {
        if (hit_rate_ == 0)
            return matches_;
        return searched_ / hit_rate_;
    }

protected:
    void find_match (const StringPiece& line) {
        chunk *c = chunk::from_str(line.data());
        int off = line.data() - c->data;
        int lno;
        for(vector<chunk_file>::iterator it = c->files.begin();
            it != c->files.end(); it++) {
            if (off >= it->left && off < it->right) {
                searched_++;
                if (matches_.load() >= MAX_MATCHES)
                    continue;
                lno = try_match(line, it->file);
                if (lno > 0) {
                    match_result *m = new match_result({it->file, lno, line});
                    queue_.push(m);
                    if (++matches_ == 10)
                        hit_rate_ = float(searched_) / matches_;
                }
            }
        }
    }

    int try_match(const StringPiece &line, search_file *sf);

    static StringPiece find_line(const StringPiece& chunk, const StringPiece& match) {
        const char *start, *end;
        assert(match.data() >= chunk.data());
        assert(match.data() < chunk.data() + chunk.size());
        assert(match.size() < (chunk.size() - (match.data() - chunk.data())));
        start = static_cast<const char*>
            (memrchr(chunk.data(), '\n', match.data() - chunk.data()));
        if (start == NULL)
            start = chunk.data();<commit_msg>Replace another harcoded 10 with MAX_MATCHES.<commit_after>#include <stdio.h>
#include <assert.h>
#include <string.h>

#include <google/dense_hash_set>

#include <locale>
#include <list>
#include <iostream>
#include <string>
#include <atomic>

#include <re2/re2.h>

#include "smart_git.h"
#include "timer.h"
#include "thread_queue.h"
#include "thread_pool.h"

using google::dense_hash_set;
using re2::RE2;
using re2::StringPiece;
using namespace std;

#define CHUNK_SIZE (1 << 20)
#define MAX_GAP    (1 << 12)
#define MAX_MATCHES 10

struct search_file {
    string path;
    const char *ref;
    git_oid oid;
};

struct chunk_file {
    search_file *file;
    int left;
    int right;
    void expand(int l, int r) {
        left  = min(left, l);
        right = max(right, r);
    }

    bool operator<(const chunk_file& rhs) const {
        return left < rhs.left;
    }
};

struct match_result {
    search_file *file;
    int lno;
    StringPiece line;
};

#define CHUNK_MAGIC 0xC407FADE

struct chunk {
    int size;
    unsigned magic;
    vector<chunk_file> files;
    vector<chunk_file> cur_file;
    char data[0];

    chunk()
        : size(0), magic(CHUNK_MAGIC), files() {
    }

    void add_chunk_file(search_file *sf, const StringPiece &line) {
        int l = line.data() - data;
        int r = l + line.size();
        chunk_file *f = NULL;
        int min_dist = numeric_limits<int>::max(), dist;
        for (vector<chunk_file>::iterator it = cur_file.begin();
             it != cur_file.end(); it ++) {
            if (l <= it->left)
                dist = max(0, it->left - r);
            else if (r >= it->right)
                dist = max(0, l - it->right);
            else
                dist = 0;
            assert(dist == 0 || r < it->left || l > it->right);
            if (dist < min_dist) {
                min_dist = dist;
                f = &(*it);
            }
        }
        if (f && min_dist < MAX_GAP) {
            f->expand(l, r);
            return;
        }
        cur_file.push_back(chunk_file());
        chunk_file &cf = cur_file.back();
        cf.file = sf;
        cf.left = l;
        cf.right = r;
    }

    void finish_file() {
        int right = -1;
        sort(cur_file.begin(), cur_file.end());
        for (vector<chunk_file>::iterator it = cur_file.begin();
             it != cur_file.end(); it ++) {
            assert(right < it->left);
            right = max(right, it->right);
        }
        files.insert(files.end(), cur_file.begin(), cur_file.end());
        cur_file.clear();
    }

    static chunk* from_str(const char *p) {
        chunk *out = reinterpret_cast<chunk*>
            (reinterpret_cast<uintptr_t>(p) & ~(CHUNK_SIZE - 1));
        assert(out->magic == CHUNK_MAGIC);
        return out;
    }
};

#define CHUNK_SPACE  (CHUNK_SIZE - (sizeof(chunk)))

chunk *alloc_chunk() {
    void *p;
    if (posix_memalign(&p, CHUNK_SIZE, CHUNK_SIZE) != 0)
        return NULL;
    return new(p) chunk;
};

class chunk_allocator {
public:
    chunk_allocator() : current_() {
        new_chunk();
    }

    char *alloc(size_t len) {
        assert(len < CHUNK_SPACE);
        if ((current_->size + len) > CHUNK_SPACE)
            new_chunk();
        char *out = current_->data + current_->size;
        current_->size += len;
        return out;
    }

    list<chunk*>::iterator begin () {
        return chunks_.begin();
    }

    typename list<chunk*>::iterator end () {
        return chunks_.end();
    }

    chunk *current_chunk() {
        return current_;
    }

protected:
    void new_chunk() {
        current_ = alloc_chunk();
        chunks_.push_back(current_);
    }

    list<chunk*> chunks_;
    chunk *current_;
};

/*
 * We special-case data() == NULL to provide an "empty" element for
 * dense_hash_set.
 *
 * StringPiece::operator== will consider a zero-length string equal to a
 * zero-length string with a NULL data().
 */
struct eqstr {
    bool operator()(const StringPiece& lhs, const StringPiece& rhs) const {
        if (lhs.data() == NULL && rhs.data() == NULL)
            return true;
        if (lhs.data() == NULL || rhs.data() == NULL)
            return false;
        return lhs == rhs;
    }
};

struct hashstr {
    locale loc;
    size_t operator()(const StringPiece &str) const {
        const collate<char> &coll = use_facet<collate<char> >(loc);
        return coll.hash(str.data(), str.data() + str.size());
    }
};

const StringPiece empty_string(NULL, 0);

typedef dense_hash_set<StringPiece, hashstr, eqstr> string_hash;

class code_counter;

class searcher {
public:
    searcher(code_counter *cc, thread_queue<match_result*> &queue, RE2& pat) :
        cc_(cc), pat_(pat), queue_(queue),
        matches_(0), searched_(0), hit_rate_(0) {
    }

    bool operator()(const chunk *chunk) {
        if (chunk == NULL) {
            queue_.push(NULL);
            return true;
        }
        StringPiece str(chunk->data, chunk->size);
        StringPiece match;
        int pos = 0;
        while (pos < str.size()) {
            if (!pat_.Match(str, pos, str.size(), RE2::UNANCHORED, &match, 1))
                break;
            assert(memchr(match.data(), '\n', match.size()) == NULL);
            StringPiece line = find_line(str, match);
            find_match(line);
            pos = line.size() + line.data() - str.data();
        }
        return false;
    }

    int approx_matches() {
        if (hit_rate_ == 0)
            return matches_;
        return searched_ / hit_rate_;
    }

protected:
    void find_match (const StringPiece& line) {
        chunk *c = chunk::from_str(line.data());
        int off = line.data() - c->data;
        int lno;
        for(vector<chunk_file>::iterator it = c->files.begin();
            it != c->files.end(); it++) {
            if (off >= it->left && off < it->right) {
                searched_++;
                if (matches_.load() >= MAX_MATCHES)
                    continue;
                lno = try_match(line, it->file);
                if (lno > 0) {
                    match_result *m = new match_result({it->file, lno, line});
                    queue_.push(m);
                    if (++matches_ == MAX_MATCHES)
                        hit_rate_ = float(searched_) / matches_;
                }
            }
        }
    }

    int try_match(const StringPiece &line, search_file *sf);

    static StringPiece find_line(const StringPiece& chunk, const StringPiece& match) {
        const char *start, *end;
        assert(match.data() >= chunk.data());
        assert(match.data() < chunk.data() + chunk.size());
        assert(match.size() < (chunk.size() - (match.data() - chunk.data())));
        start = static_cast<const char*>
            (memrchr(chunk.data(), '\n', match.data() - chunk.data()));
        if (start == NULL)
            start = chunk.data();
{'fraction_non_alphanumeric': 0.1441500620392672, 'fraction_numerical': 0.005547040362017371, 'mean_word_length': 2.0240564996689474, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 117, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6384368', 'n_tokens_mistral': 4486, 'n_tokens_neox': 4399, 'n_words': 1064}