<commit_before>/*****************************************************************************
 * Licensed to Qualys, Inc. (QUALYS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * QUALYS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ****************************************************************************/

/**
 * @file
 * @brief IronBee --- PCRE Module
 *
 * @author <NAME> <<EMAIL>>
 */

#include <ironbee/bytestr.h>
#include <ironbee/capture.h>
#include <ironbee/cfgmap.h>
#include <ironbee/context.h>
#include <ironbee/engine.h>
#include <ironbee/escape.h>
#include <ironbee/field.h>
#include <ironbee/mm.h>
#include <ironbee/module.h>
#include <ironbee/operator.h>
#include <ironbee/rule_engine.h>
#include <ironbee/string.h>
#include <ironbee/transformation.h>
#include <ironbee/type_convert.h>
#include <ironbee/util.h>

#include <pcre.h>

#include <assert.h>
#include <ctype.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <time.h>

/* Define the module name as well as a string version of it. */
#define MODULE_NAME        pcre
#define MODULE_NAME_STR    IB_XSTRINGIFY(MODULE_NAME)

/* How many matches will PCRE find and populate. */
#define MATCH_MAX 10

/* Port forward the pcre constant PCRE_PARTIAL as PCRE_PARTIAL_SOFT. */
#ifndef PCRE_PARTIAL_SOFT
#define PCRE_PARTIAL_SOFT PCRE_PARTIAL
#endif

/**
 * From pcreapi man page.
 */
#define WORKSPACE_SIZE_MIN     (20)

/**
 * Build a reasonable buffer size.
 */
#define WORKSPACE_SIZE_DEFAULT (WORKSPACE_SIZE_MIN * 10)

/* Define the public module symbol. */
IB_MODULE_DECLARE();

/**
 * Module Configuration Structure.
 */
struct modpcre_cfg_t {
    ib_num_t       study;                 /**< Bool: Study compiled regexs */
    ib_num_t       use_jit;               /**< Bool: Use JIT if available */
    ib_num_t       match_limit;           /**< Match limit */
    ib_num_t       match_limit_recursion; /**< Match recursion depth limit */
    ib_num_t       jit_stack_start;       /**< Starting JIT stack size */
    ib_num_t       jit_stack_max;         /**< Max JIT stack size */
    ib_num_t       dfa_workspace_size;    /**< Size of DFA workspace */
};
typedef struct modpcre_cfg_t modpcre_cfg_t;

/**
 * Internal representation of PCRE compiled patterns.
 */
struct modpcre_cpat_data_t {
    ib_module_t         *module;          /**< Pointer to this module. */
    pcre                *cpatt;           /**< Compiled pattern */
    pcre_extra          *edata;           /**< PCRE Study data */
    const char          *patt;            /**< Regex pattern text */
    bool                 is_dfa;          /**< Is this a DFA? */
    bool                 is_jit;          /**< Is this JIT compiled? */
    int                  dfa_ws_size;     /**< Size of DFA workspace */
};
typedef struct modpcre_cpat_data_t modpcre_cpat_data_t;

/**
 * PCRE and DFA rule data types are an alias for the compiled pattern structure.
 */
struct modpcre_operator_data_t {
    modpcre_cpat_data_t *cpdata;          /**< Compiled pattern data */
    const char          *id;              /**< ID for DFA rules */
};
typedef struct modpcre_operator_data_t modpcre_operator_data_t;

/* Instantiate a module global configuration. */
static modpcre_cfg_t modpcre_global_cfg = {
    1,                     /* study. */
    1,                     /* use_jit. */
    5000,                  /* match_limit. */
    5000,                  /* match_limit_recursion. */
    32 * 1024,             /* jit_stack_start. */
    1000 * 1024,           /* jit_stack_max. */
    WORKSPACE_SIZE_DEFAULT /* dfa_workspace_size. */
};

/* State information for a PCRE work common to all pcre operators in a tx. */
struct pcre_tx_data_t {
    pcre_jit_stack *stack;
    ib_hash_t      *dfa_workspace_hash;
};
typedef struct pcre_tx_data_t pcre_tx_data_t;

#ifdef PCRE_HAVE_JIT
static void pcre_jit_stack_cleanup(void *stack) {
    assert(stack != NULL);
    pcre_jit_stack_free((pcre_jit_stack *)stack);
}
#endif

/**
 * Get or create an ib_hash_t inside of @c tx for storing dfa rule data.
 *
 * The hash is stored at the key @c HASH_NAME_STR.
 *
 * @param[in] m  PCRE module.
 * @param[in] tx The transaction containing @c tx->data which holds
 *            the @a operator_data object.
 * @param[out] hash The fetched or created rule data hash. This is set
 *             to NULL on failure.
 *
 * @return
 *   - IB_OK on success.
 *   - IB_EALLOC on allocation failure
 */
static
ib_status_t get_or_create_operator_data(
    const ib_module_t  *m,
    ib_tx_t            *tx,
    pcre_tx_data_t    **data
)
{
    assert(tx);

    ib_status_t     rc;
    pcre_tx_data_t *data_tmp;

    /* Get or create the hash that contains the rule data. */
    rc = ib_tx_get_module_data(tx, m, data);
    if ( (rc == IB_OK) && (*data != NULL) ) {
        return IB_OK;
    }

    data_tmp = ib_mm_alloc(tx->mm, sizeof(*data_tmp));
    if (data_tmp == NULL) {
        return IB_EALLOC;
    }

    /* Create the DFA Hash. */
    {
        rc = ib_hash_create(&data_tmp->dfa_workspace_hash, tx->mm);
        if (rc != IB_OK) {
            return rc;
        }

        rc = ib_tx_set_module_data(tx, m, data_tmp);
        if (rc != IB_OK) {
            return rc;
        }
    }

    /* Initialize the PCRE JIT Stack for the TX. */
#ifdef PCRE_HAVE_JIT
    {
        modpcre_cfg_t  *config;

        rc = ib_context_module_config(tx->ctx, m, &config);
        if (rc != IB_OK) {
            ib_log_error_tx(tx, "Cannot fetch module config for pcre.");
            return rc;
        }

        data_tmp->stack = pcre_jit_stack_alloc(
            config->jit_stack_start,
            config->jit_stack_max
        );
        /* A null stack is extremely unexpected, but not fatal.
         * JIT can use a callstack in a threadsafe way. */
        if (data_tmp->stack == NULL) {
            ib_log_error_warn(
                tx,
                "Could not allocate a pcre JIT stack: min=%d max=%d",
                config->jit_stack_start,
                config->jit_stack_max
            );
        }
        else {
            rc = ib_mm_register_cleanup(
                tx->mm,
                pcre_jit_stack_cleanup,
                data_tmp->stack
            );
            if (rc != IB_OK) {
                pcre_jit_stack_free(data_tmp->stack);
                return rc;
            }
        }
    }
#endif

    *data = data_tmp;
<commit_msg>pcre.c: Fixing logging statement error. RNS-1496.<commit_after>/*****************************************************************************
 * Licensed to Qualys, Inc. (QUALYS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * QUALYS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ****************************************************************************/

/**
 * @file
 * @brief IronBee --- PCRE Module
 *
 * @author <NAME> <<EMAIL>>
 */

#include <ironbee/bytestr.h>
#include <ironbee/capture.h>
#include <ironbee/cfgmap.h>
#include <ironbee/context.h>
#include <ironbee/engine.h>
#include <ironbee/escape.h>
#include <ironbee/field.h>
#include <ironbee/mm.h>
#include <ironbee/module.h>
#include <ironbee/operator.h>
#include <ironbee/rule_engine.h>
#include <ironbee/string.h>
#include <ironbee/transformation.h>
#include <ironbee/type_convert.h>
#include <ironbee/util.h>

#include <pcre.h>

#include <assert.h>
#include <ctype.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <strings.h>
#include <time.h>

/* Define the module name as well as a string version of it. */
#define MODULE_NAME        pcre
#define MODULE_NAME_STR    IB_XSTRINGIFY(MODULE_NAME)

/* How many matches will PCRE find and populate. */
#define MATCH_MAX 10

/* Port forward the pcre constant PCRE_PARTIAL as PCRE_PARTIAL_SOFT. */
#ifndef PCRE_PARTIAL_SOFT
#define PCRE_PARTIAL_SOFT PCRE_PARTIAL
#endif

/**
 * From pcreapi man page.
 */
#define WORKSPACE_SIZE_MIN     (20)

/**
 * Build a reasonable buffer size.
 */
#define WORKSPACE_SIZE_DEFAULT (WORKSPACE_SIZE_MIN * 10)

/* Define the public module symbol. */
IB_MODULE_DECLARE();

/**
 * Module Configuration Structure.
 */
struct modpcre_cfg_t {
    ib_num_t       study;                 /**< Bool: Study compiled regexs */
    ib_num_t       use_jit;               /**< Bool: Use JIT if available */
    ib_num_t       match_limit;           /**< Match limit */
    ib_num_t       match_limit_recursion; /**< Match recursion depth limit */
    ib_num_t       jit_stack_start;       /**< Starting JIT stack size */
    ib_num_t       jit_stack_max;         /**< Max JIT stack size */
    ib_num_t       dfa_workspace_size;    /**< Size of DFA workspace */
};
typedef struct modpcre_cfg_t modpcre_cfg_t;

/**
 * Internal representation of PCRE compiled patterns.
 */
struct modpcre_cpat_data_t {
    ib_module_t         *module;          /**< Pointer to this module. */
    pcre                *cpatt;           /**< Compiled pattern */
    pcre_extra          *edata;           /**< PCRE Study data */
    const char          *patt;            /**< Regex pattern text */
    bool                 is_dfa;          /**< Is this a DFA? */
    bool                 is_jit;          /**< Is this JIT compiled? */
    int                  dfa_ws_size;     /**< Size of DFA workspace */
};
typedef struct modpcre_cpat_data_t modpcre_cpat_data_t;

/**
 * PCRE and DFA rule data types are an alias for the compiled pattern structure.
 */
struct modpcre_operator_data_t {
    modpcre_cpat_data_t *cpdata;          /**< Compiled pattern data */
    const char          *id;              /**< ID for DFA rules */
};
typedef struct modpcre_operator_data_t modpcre_operator_data_t;

/* Instantiate a module global configuration. */
static modpcre_cfg_t modpcre_global_cfg = {
    1,                     /* study. */
    1,                     /* use_jit. */
    5000,                  /* match_limit. */
    5000,                  /* match_limit_recursion. */
    32 * 1024,             /* jit_stack_start. */
    1000 * 1024,           /* jit_stack_max. */
    WORKSPACE_SIZE_DEFAULT /* dfa_workspace_size. */
};

/* State information for a PCRE work common to all pcre operators in a tx. */
struct pcre_tx_data_t {
    pcre_jit_stack *stack;
    ib_hash_t      *dfa_workspace_hash;
};
typedef struct pcre_tx_data_t pcre_tx_data_t;

#ifdef PCRE_HAVE_JIT
static void pcre_jit_stack_cleanup(void *stack) {
    assert(stack != NULL);
    pcre_jit_stack_free((pcre_jit_stack *)stack);
}
#endif

/**
 * Get or create an ib_hash_t inside of @c tx for storing dfa rule data.
 *
 * The hash is stored at the key @c HASH_NAME_STR.
 *
 * @param[in] m  PCRE module.
 * @param[in] tx The transaction containing @c tx->data which holds
 *            the @a operator_data object.
 * @param[out] hash The fetched or created rule data hash. This is set
 *             to NULL on failure.
 *
 * @return
 *   - IB_OK on success.
 *   - IB_EALLOC on allocation failure
 */
static
ib_status_t get_or_create_operator_data(
    const ib_module_t  *m,
    ib_tx_t            *tx,
    pcre_tx_data_t    **data
)
{
    assert(tx);

    ib_status_t     rc;
    pcre_tx_data_t *data_tmp;

    /* Get or create the hash that contains the rule data. */
    rc = ib_tx_get_module_data(tx, m, data);
    if ( (rc == IB_OK) && (*data != NULL) ) {
        return IB_OK;
    }

    data_tmp = ib_mm_alloc(tx->mm, sizeof(*data_tmp));
    if (data_tmp == NULL) {
        return IB_EALLOC;
    }

    /* Create the DFA Hash. */
    {
        rc = ib_hash_create(&data_tmp->dfa_workspace_hash, tx->mm);
        if (rc != IB_OK) {
            return rc;
        }

        rc = ib_tx_set_module_data(tx, m, data_tmp);
        if (rc != IB_OK) {
            return rc;
        }
    }

    /* Initialize the PCRE JIT Stack for the TX. */
#ifdef PCRE_HAVE_JIT
    {
        modpcre_cfg_t  *config;

        rc = ib_context_module_config(tx->ctx, m, &config);
        if (rc != IB_OK) {
            ib_log_error_tx(tx, "Cannot fetch module config for pcre.");
            return rc;
        }

        data_tmp->stack = pcre_jit_stack_alloc(
            config->jit_stack_start,
            config->jit_stack_max
        );
        /* A null stack is extremely unexpected, but not fatal.
         * JIT can use a callstack in a threadsafe way. */
        if (data_tmp->stack == NULL) {
            ib_log_warning_tx(
                tx,
                "Could not allocate a pcre JIT stack: min=%d max=%d",
                (int)config->jit_stack_start,
                (int)config->jit_stack_max
            );
        }
        else {
            rc = ib_mm_register_cleanup(
                tx->mm,
                pcre_jit_stack_cleanup,
                data_tmp->stack
            );
            if (rc != IB_OK) {
                pcre_jit_stack_free(data_tmp->stack);
                return rc;
            }
        }
    }
#endif

    *data = data_tmp;

{'fraction_non_alphanumeric': 0.1229588963963964, 'fraction_numerical': 0.005067567567567568, 'mean_word_length': 2.281524249422633, 'pattern_counts': {'":': 0, '<': 89, '<?xml version=': 0, '>': 85, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1402955', 'n_tokens_mistral': 4674, 'n_tokens_neox': 4481, 'n_words': 1273}