<commit_before>
/*
 * Copyright 2012 Yichun "agentzh" Zhang
 * Copyright 2007-2009 <NAME>.  All Rights Reserved.
 * Use of this source code is governed by a BSD-style
 * license that can be found in the LICENSE file.
 */


#ifndef DDEBUG
#define DDEBUG 0
#endif
#include <sregex/ddebug.h>


#include <sregex/sre_vm_pike.h>
#include <sregex/sre_capture.h>


#define sre_vm_pike_free_thread(ctx, t)                                     \
    (t)->next = (ctx)->free_threads;                                        \
    (ctx)->free_threads = t;


enum {
    SRE_VM_PIKE_SEEN_WORD = 1
};


typedef struct sre_vm_pike_thread_s  sre_vm_pike_thread_t;

struct sre_vm_pike_thread_s {
    sre_instruction_t       *pc;
    sre_capture_t           *capture;
    sre_vm_pike_thread_t    *next;
    unsigned                 seen_word; /* :1 */
};


typedef struct {
    sre_vm_pike_thread_t     *head;
    sre_vm_pike_thread_t    **next;
} sre_vm_pike_thread_list_t;


struct sre_vm_pike_ctx_s {
    unsigned                 tag;
    unsigned                 processed_bytes;
    u_char                  *buffer;
    sre_pool_t              *pool;
    sre_program_t           *program;
    sre_capture_t           *matched;
    sre_capture_t           *free_capture;
    sre_vm_pike_thread_t    *free_threads;

    int                     *ovector;
    unsigned                 ovecsize;

    sre_vm_pike_thread_list_t       *current_threads;
    sre_vm_pike_thread_list_t       *next_threads;

    unsigned                 first_buf; /* :1 */
} ;


static sre_vm_pike_thread_list_t *sre_vm_pike_thread_list_create(
    sre_pool_t *pool, int size);
static int sre_vm_pike_add_thread(sre_vm_pike_ctx_t *ctx,
    sre_vm_pike_thread_list_t *l, sre_instruction_t *pc, sre_capture_t *capture,
    int pos);
static void sre_vm_pike_prepare_temp_captures(sre_vm_pike_ctx_t *ctx);


sre_vm_pike_ctx_t *
sre_vm_pike_init(sre_pool_t *pool, sre_program_t *prog, int *ovector,
    unsigned ovecsize)
{
    unsigned                         len;
    sre_vm_pike_ctx_t               *ctx;
    sre_vm_pike_thread_list_t       *clist, *nlist;

    ctx = sre_palloc(pool, sizeof(sre_vm_pike_ctx_t));
    if (ctx == NULL) {
        return NULL;
    }

    ctx->pool = pool;
    ctx->program = prog;

    ctx->tag = prog->tag + 1;

    len = prog->len;

    clist = sre_vm_pike_thread_list_create(pool, len);
    if (clist == NULL) {
        return NULL;
    }

    ctx->current_threads = clist;

    nlist = sre_vm_pike_thread_list_create(pool, len);
    if (nlist == NULL) {
        return NULL;
    }

    ctx->next_threads = nlist;

    ctx->program = prog;
    ctx->pool = pool;
    ctx->free_capture = NULL;
    ctx->free_threads = NULL;
    ctx->processed_bytes = 0;
    ctx->matched = NULL;

    ctx->ovecsize = ovecsize;
    ctx->ovector = ovector;

    ctx->first_buf = 1;

    return ctx;
}


void
sre_vm_pike_finalize(sre_vm_pike_ctx_t *ctx)
{
}


int
sre_vm_pike_exec(sre_vm_pike_ctx_t *ctx, u_char *input, size_t size,
    unsigned eof)
{
    u_char                    *sp, *last;
    unsigned                   i;
    unsigned                   in;
    sre_pool_t                *pool;
    sre_program_t             *prog;
    sre_capture_t             *cap, *matched;
    sre_vm_range_t            *range;
    sre_instruction_t         *pc;
    sre_vm_pike_thread_t      *t;
    sre_vm_pike_thread_list_t *clist, *nlist, *tmp;
    sre_vm_pike_thread_list_t  list;

    pool = ctx->pool;
    prog = ctx->program;
    clist = ctx->current_threads;
    nlist = ctx->next_threads;
    matched = ctx->matched;

    ctx->buffer = input;

    if (ctx->first_buf) {
        ctx->first_buf = 0;

        cap = sre_capture_create(pool, ctx->ovecsize, 1);
        if (cap == NULL) {
            return SRE_ERROR;
        }

        if (sre_vm_pike_add_thread(ctx, clist, prog->start, cap, 0) != SRE_OK) {
            prog->tag = ctx->tag;
            return SRE_ERROR;
        }
    }<commit_msg>feature: the same ngx_program_t data structure can now be shared among multiple different coroutines within a single OS thread.<commit_after>
/*
 * Copyright 2012 Yichun "agentzh" Zhang
 * Copyright 2007-2009 <NAME>.  All Rights Reserved.
 * Use of this source code is governed by a BSD-style
 * license that can be found in the LICENSE file.
 */


#ifndef DDEBUG
#define DDEBUG 0
#endif
#include <sregex/ddebug.h>


#include <sregex/sre_vm_pike.h>
#include <sregex/sre_capture.h>


#define sre_vm_pike_free_thread(ctx, t)                                     \
    (t)->next = (ctx)->free_threads;                                        \
    (ctx)->free_threads = t;


enum {
    SRE_VM_PIKE_SEEN_WORD = 1
};


typedef struct sre_vm_pike_thread_s  sre_vm_pike_thread_t;

struct sre_vm_pike_thread_s {
    sre_instruction_t       *pc;
    sre_capture_t           *capture;
    sre_vm_pike_thread_t    *next;
    unsigned                 seen_word; /* :1 */
};


typedef struct {
    sre_vm_pike_thread_t     *head;
    sre_vm_pike_thread_t    **next;
} sre_vm_pike_thread_list_t;


struct sre_vm_pike_ctx_s {
    unsigned                 tag;
    unsigned                 processed_bytes;
    u_char                  *buffer;
    sre_pool_t              *pool;
    sre_program_t           *program;
    sre_capture_t           *matched;
    sre_capture_t           *free_capture;
    sre_vm_pike_thread_t    *free_threads;

    int                     *ovector;
    unsigned                 ovecsize;

    sre_vm_pike_thread_list_t       *current_threads;
    sre_vm_pike_thread_list_t       *next_threads;

    unsigned                 first_buf; /* :1 */
} ;


static sre_vm_pike_thread_list_t *sre_vm_pike_thread_list_create(
    sre_pool_t *pool, int size);
static int sre_vm_pike_add_thread(sre_vm_pike_ctx_t *ctx,
    sre_vm_pike_thread_list_t *l, sre_instruction_t *pc, sre_capture_t *capture,
    int pos);
static void sre_vm_pike_prepare_temp_captures(sre_vm_pike_ctx_t *ctx);


sre_vm_pike_ctx_t *
sre_vm_pike_init(sre_pool_t *pool, sre_program_t *prog, int *ovector,
    unsigned ovecsize)
{
    unsigned                         len;
    sre_vm_pike_ctx_t               *ctx;
    sre_vm_pike_thread_list_t       *clist, *nlist;

    ctx = sre_palloc(pool, sizeof(sre_vm_pike_ctx_t));
    if (ctx == NULL) {
        return NULL;
    }

    ctx->pool = pool;
    ctx->program = prog;

    len = prog->len;

    clist = sre_vm_pike_thread_list_create(pool, len);
    if (clist == NULL) {
        return NULL;
    }

    ctx->current_threads = clist;

    nlist = sre_vm_pike_thread_list_create(pool, len);
    if (nlist == NULL) {
        return NULL;
    }

    ctx->next_threads = nlist;

    ctx->program = prog;
    ctx->pool = pool;
    ctx->free_capture = NULL;
    ctx->free_threads = NULL;
    ctx->processed_bytes = 0;
    ctx->matched = NULL;

    ctx->ovecsize = ovecsize;
    ctx->ovector = ovector;

    ctx->first_buf = 1;

    return ctx;
}


void
sre_vm_pike_finalize(sre_vm_pike_ctx_t *ctx)
{
}


int
sre_vm_pike_exec(sre_vm_pike_ctx_t *ctx, u_char *input, size_t size,
    unsigned eof)
{
    u_char                    *sp, *last;
    unsigned                   i;
    unsigned                   in;
    sre_pool_t                *pool;
    sre_program_t             *prog;
    sre_capture_t             *cap, *matched;
    sre_vm_range_t            *range;
    sre_instruction_t         *pc;
    sre_vm_pike_thread_t      *t;
    sre_vm_pike_thread_list_t *clist, *nlist, *tmp;
    sre_vm_pike_thread_list_t  list;

    pool = ctx->pool;
    prog = ctx->program;
    clist = ctx->current_threads;
    nlist = ctx->next_threads;
    matched = ctx->matched;

    ctx->buffer = input;

    if (ctx->first_buf) {
        ctx->first_buf = 0;

        cap = sre_capture_create(pool, ctx->ovecsize, 1);
        if (cap == NULL) {
            return SRE_ERROR;
        }

        ctx->tag = prog->tag + 1;
        if (sre_vm_pike_add_thread(ctx, clist, prog->start, cap, 0) != SRE_OK) {
            prog->tag = ctx->tag;
            return SRE_ERROR;
        }

    } else {
        ctx->tag = prog->tag;
    }
{'fraction_non_alphanumeric': 0.09237989378782265, 'fraction_numerical': 0.005434111399283685, 'mean_word_length': 2.0837776085300836, 'pattern_counts': {'":': 0, '<': 11, '<?xml version=': 0, '>': 75, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 4, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '2850354', 'n_tokens_mistral': 3117, 'n_tokens_neox': 2890, 'n_words': 569}