<commit_before>/**
 * @brief A character based analyzer
 * @author Wei
 * @date 2010.08.24
 */

#include <la/analyzer/CharAnalyzer.h>

#include <util/ustring/UString.h>

using namespace izenelib::util;
using namespace std;

namespace la
{

CharAnalyzer::CharAnalyzer()
    : Analyzer(),
      isSeparateAll( true )
{
}

CharAnalyzer::~CharAnalyzer()
{
}

void CharAnalyzer::setSeparateAll( bool flag )
{
    isSeparateAll = flag;
}

int CharAnalyzer::analyze_impl( const Term& input, void* data, HookType func )
{
    if( isSeparateAll == true )
        return separate_all( input, data, func );
    return separate_part( input, data, func );
}

int CharAnalyzer::separate_part( const Term& input, void* data, HookType func )
{
    int offset = 0;

    bool digit = false;
    bool alpha = false;
    int begin = 0;
    int end = 0;
    for( size_t i = 0; i < input.text_.length(); ++i )
    {
        UString::CharT ch = input.text_.at(i);

        if(UString::isThisDigitChar(ch))
        {
            if(!digit)
            {
                if(alpha)
                {
                    func(data, input.text_.c_str()+begin, end-begin+1, offset++, Term::EnglishPOS, Term::AND , 0, false);
                    alpha = false;
                }
                begin = i;
                digit = true;
            }
            end = i;
            continue;
        }
        else
        {
            if(digit)
            {
                func(data, input.text_.c_str()+begin, end-begin+1, offset++, Term::DigitPOS, Term::AND , 0, false);
                digit = false;
            }
        }
<commit_msg>divide full-shaped nubmers in CharAnalyzer::separate_part().<commit_after>/**
 * @brief A character based analyzer
 * @author Wei
 * @date 2010.08.24
 */

#include <la/analyzer/CharAnalyzer.h>

#include <util/ustring/UString.h>

using namespace izenelib::util;
using namespace std;

namespace la
{

CharAnalyzer::CharAnalyzer()
    : Analyzer(),
      isSeparateAll( true )
{
}

CharAnalyzer::~CharAnalyzer()
{
}

void CharAnalyzer::setSeparateAll( bool flag )
{
    isSeparateAll = flag;
}

int CharAnalyzer::analyze_impl( const Term& input, void* data, HookType func )
{
    if( isSeparateAll == true )
        return separate_all( input, data, func );
    return separate_part( input, data, func );
}

int CharAnalyzer::separate_part( const Term& input, void* data, HookType func )
{
    int offset = 0;

    bool digit = false;
    bool alpha = false;
    int begin = 0;
    int end = 0;
    for( size_t i = 0; i < input.text_.length(); ++i )
    {
        UString::CharT ch = input.text_.at(i);

        if(UString::isThisNumericChar(ch))
        {
            if(!digit)
            {
                if(alpha)
                {
                    func(data, input.text_.c_str()+begin, end-begin+1, offset++, Term::EnglishPOS, Term::AND , 0, false);
                    alpha = false;
                }
                begin = i;
                digit = true;
            }
            end = i;
            continue;
        }
        else
        {
            if(digit)
            {
                func(data, input.text_.c_str()+begin, end-begin+1, offset++, Term::DigitPOS, Term::AND , 0, false);
                digit = false;
            }
        }

{'fraction_non_alphanumeric': 0.1262580054894785, 'fraction_numerical': 0.009759072888075633, 'mean_word_length': 1.8646288209606987, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3858873', 'n_tokens_mistral': 1096, 'n_tokens_neox': 1029, 'n_words': 246}