<commit_before>/*****************************************************************************
 * Licensed to Qualys, Inc. (QUALYS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * QUALYS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ****************************************************************************/

/**
 * @file
 * @brief IronBee - String related functions
 * @author <NAME> <<EMAIL>>
 */

#include "ironbee_config_auto.h"

#include <ironbee/mpool.h>
#include <ironbee/string.h>
#include <ironbee/types.h>

#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

/**
 * Count the amount of whitespace in a string function
 *
 * @param[in] minlen Minimum length of a run of whitespace to count
 * @param[in] data String to analyze
 * @param[in] dlen Length of @a dlen
 * @param[out] count Number of whitespace characters of runs > whitespace
 * @param[out] other Number of whitespace characters that are not ' '.
 */
typedef void (* count_fn_t)(size_t minlen,
                            const uint8_t *data,
                            size_t dlen,
                            size_t *count,
                            size_t *other);

/**
 * In-place whitespace removal/compression function
 *
 * @param[in,out] buf Buffer to operate on
 * @param[in] dlen_in Input length
 * @param[out] dlen_out Length after whitespace removal
 * @param[out] result Result flags (@c IB_STRFLAG_xxx)
 *
 * @returns Status code.
 */
typedef ib_status_t (* inplace_fn_t)(uint8_t *buf,
                                     size_t dlen_in,
                                     size_t *dlen_out,
                                     ib_flags_t *result);
/**
 * Non-in-place whitespace removal/compression function
 *
 * @param[in] data_in Input buffer
 * @param[in] dlen_in Length of @a data_in
 * @param[out] data_out Output buffer
 * @param[in] dlen_out Length of @a data_out
 *
 * @returns Status code.
 */
typedef ib_status_t (* outplace_fn_t)(const uint8_t *data_in,
                                      size_t dlen_in,
                                      uint8_t *data_out,
                                      size_t dlen_out);

/**
 * Count the amount of whitespace in a string
 *
 * @param[in] force_other_zero If true, force @a other to zero
 * @param[in] minlen Minimum length of a run of whitespace to count
 * @param[in] data String to analyze
 * @param[in] dlen Length of @a dlen
 * @param[out] count Number of whitespace characters of runs > whitespace
 * @param[out] other Number of whitespace characters that are not ' '.
 */
static void ws_count(bool force_other_zero,
                     size_t minlen,
                     const uint8_t *data,
                     size_t dlen,
                     size_t *count,
                     size_t *other)
{
    const uint8_t *end;
    size_t icount = 0;     /* Internal count */
    size_t iother = 0;     /* Internal other */
    size_t runlen = 0;

    /* Loop through the whole string */
    end = data + dlen;
    while (data <= end) {
        uint8_t c = *data;
        if (isspace(c) == 0) {
            runlen = 0;
        }
        else {
            ++runlen;
            if (runlen >= minlen) {
                ++icount;
            }
            if (c != ' ') {
                ++iother;
            }
        }
        ++data;
    }
<commit_msg>Whitespace removal: Fix off-by-one bug in counting spaces.<commit_after>/*****************************************************************************
 * Licensed to Qualys, Inc. (QUALYS) under one or more
 * contributor license agreements.  See the NOTICE file distributed with
 * this work for additional information regarding copyright ownership.
 * QUALYS licenses this file to You under the Apache License, Version 2.0
 * (the "License"); you may not use this file except in compliance with
 * the License.  You may obtain a copy of the License at
 *
 *     http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 ****************************************************************************/

/**
 * @file
 * @brief IronBee - String related functions
 * @author <NAME> <<EMAIL>>
 */

#include "ironbee_config_auto.h"

#include <ironbee/mpool.h>
#include <ironbee/string.h>
#include <ironbee/types.h>

#include <assert.h>
#include <ctype.h>
#include <errno.h>
#include <limits.h>
#include <stdbool.h>
#include <stdio.h>
#include <stdlib.h>

/**
 * Count the amount of whitespace in a string function
 *
 * @param[in] minlen Minimum length of a run of whitespace to count
 * @param[in] data String to analyze
 * @param[in] dlen Length of @a dlen
 * @param[out] count Number of whitespace characters of runs > whitespace
 * @param[out] other Number of whitespace characters that are not ' '.
 */
typedef void (* count_fn_t)(size_t minlen,
                            const uint8_t *data,
                            size_t dlen,
                            size_t *count,
                            size_t *other);

/**
 * In-place whitespace removal/compression function
 *
 * @param[in,out] buf Buffer to operate on
 * @param[in] dlen_in Input length
 * @param[out] dlen_out Length after whitespace removal
 * @param[out] result Result flags (@c IB_STRFLAG_xxx)
 *
 * @returns Status code.
 */
typedef ib_status_t (* inplace_fn_t)(uint8_t *buf,
                                     size_t dlen_in,
                                     size_t *dlen_out,
                                     ib_flags_t *result);
/**
 * Non-in-place whitespace removal/compression function
 *
 * @param[in] data_in Input buffer
 * @param[in] dlen_in Length of @a data_in
 * @param[out] data_out Output buffer
 * @param[in] dlen_out Length of @a data_out
 *
 * @returns Status code.
 */
typedef ib_status_t (* outplace_fn_t)(const uint8_t *data_in,
                                      size_t dlen_in,
                                      uint8_t *data_out,
                                      size_t dlen_out);

/**
 * Count the amount of whitespace in a string
 *
 * @param[in] force_other_zero If true, force @a other to zero
 * @param[in] minlen Minimum length of a run of whitespace to count
 * @param[in] data String to analyze
 * @param[in] dlen Length of @a dlen
 * @param[out] count Number of whitespace characters of runs > whitespace
 * @param[out] other Number of whitespace characters that are not ' '.
 */
static void ws_count(bool force_other_zero,
                     size_t minlen,
                     const uint8_t *data,
                     size_t dlen,
                     size_t *count,
                     size_t *other)
{
    const uint8_t *end;
    size_t icount = 0;     /* Internal count */
    size_t iother = 0;     /* Internal other */
    size_t runlen = 0;

    /* Loop through the whole string */
    end = data + dlen;
    while (data < end) {
        uint8_t c = *data;
        if (isspace(c) == 0) {
            runlen = 0;
        }
        else {
            ++runlen;
            if (runlen >= minlen) {
                ++icount;
            }
            if (c != ' ') {
                ++iother;
            }
        }
        ++data;
    }

{'fraction_non_alphanumeric': 0.1280127889818003, 'fraction_numerical': 0.003935071323167732, 'mean_word_length': 2.226100753669179, 'pattern_counts': {'":': 0, '<': 31, '<?xml version=': 0, '>': 35, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 12, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3657604', 'n_tokens_mistral': 2361, 'n_tokens_neox': 2270, 'n_words': 801}