<commit_before>/*
 * File:  bulkload.cpp
 * Copyright (C) 2004 ISP RAS
 * The Institute for System Programming of the Russian Academy of Sciences
 */

#include <math.h>
#include <map>
#include <ostream>

#include "bulkload.h"

#include "tr/structures/metadata.h"
#include "expat.h"
#include "common/xptr.h"
#include "tr/mo/mo.h"

#ifdef SE_ENABLE_FTSEARCH
#include "tr/ft/ft_index_data.h"
#endif

#include "tr/updates/updates.h"
#include "tr/structures/producer.h"
#include "tr/executor/base/SCElementProducer.h"

#define BUFFER_SIZE (16*1024)
#define MAX_QNAME 1024
#define TEXT_BUFFER_SIZE PAGE_SIZE
#define SEPARATOR    '>'

static
bool whitespaceOnly(const text_source_t &x) {
    U_ASSERT(x.type == text_source_t::text_mem);

    for (size_t i = 0; i < (size_t) x.size; i++) {
        int c = x.u.cstr[i];

        if (!isspace(c)) {
            return false;
        }
    }

    return true;
}

static
text_source_t clearLeftSpaces(const text_source_t &x) {
    U_ASSERT(x.type == text_source_t::text_mem);

    text_source_t result = NULL_TEXT; /* NULL_TEXT is text_mem ! */

    for (size_t i = 0; i < (size_t) x.size; i++) {
        int c = x.u.cstr[i];

        if (!isspace(c)) {
            result.u.cstr = x.u.cstr + i;
            result.size = x.size - i;
            return result;
        }
    }

    U_ASSERT(result.type == text_source_t::text_mem);
    return result;
}

static
text_source_t getRightSpaces(const text_source_t &x) {
    U_ASSERT(x.type == text_source_t::text_mem);

    text_source_t result = NULL_TEXT; /* NULL_TEXT is text_mem ! */

    for (size_t i = (size_t) x.size; i > 0; i--) {
        int c = x.u.cstr[i-1];

        if (!isspace(c)) {
            result.u.cstr = x.u.cstr + i;
            result.size = x.u.cstr + x.size - result.u.cstr;
            return result;<commit_msg>MSVS debug build compiler bug workaround for isspace intrisic.<commit_after>/*
 * File:  bulkload.cpp
 * Copyright (C) 2004 ISP RAS
 * The Institute for System Programming of the Russian Academy of Sciences
 */

#include <math.h>
#include <map>
#include <ostream>

#include "bulkload.h"

#include "tr/structures/metadata.h"
#include "expat.h"
#include "common/xptr.h"
#include "tr/mo/mo.h"

#ifdef SE_ENABLE_FTSEARCH
#include "tr/ft/ft_index_data.h"
#endif

#include "tr/updates/updates.h"
#include "tr/structures/producer.h"
#include "tr/executor/base/SCElementProducer.h"

#define BUFFER_SIZE (16*1024)
#define MAX_QNAME 1024
#define TEXT_BUFFER_SIZE PAGE_SIZE
#define SEPARATOR    '>'

static
bool whitespaceOnly(const text_source_t &x) {
    U_ASSERT(x.type == text_source_t::text_mem);

    for (size_t i = 0; i < (size_t) x.size; i++) {
        int c = x.u.cstr[i];

        if (!isspace(c & 0x7f)) {
            return false;
        }
    }

    return true;
}

static
text_source_t clearLeftSpaces(const text_source_t &x) {
    U_ASSERT(x.type == text_source_t::text_mem);

    text_source_t result = NULL_TEXT; /* NULL_TEXT is text_mem ! */

    for (size_t i = 0; i < (size_t) x.size; i++) {
        int c = x.u.cstr[i];

        if (!isspace(c & 0x7f)) {
            result.u.cstr = x.u.cstr + i;
            result.size = x.size - i;
            return result;
        }
    }

    U_ASSERT(result.type == text_source_t::text_mem);
    return result;
}

static
text_source_t getRightSpaces(const text_source_t &x) {
    U_ASSERT(x.type == text_source_t::text_mem);

    text_source_t result = NULL_TEXT; /* NULL_TEXT is text_mem ! */

    for (size_t i = (size_t) x.size; i > 0; i--) {
        int c = x.u.cstr[i-1];

        if (!isspace(c & 0x7f)) {
            result.u.cstr = x.u.cstr + i;
            result.size = x.u.cstr + x.size - result.u.cstr;
            return result;
{'fraction_non_alphanumeric': 0.13932221624529317, 'fraction_numerical': 0.011296395911780527, 'mean_word_length': 2.6968190854870775, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 13, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3720389', 'n_tokens_mistral': 1474, 'n_tokens_neox': 1412, 'n_words': 318}