<commit_before>

#if HAVE_CONFIG_H
#include "config.h"
#endif

#if YAZ_HAVE_ICU

#include <yaz/yconfig.h>

#include <yaz/stemmer.h>

#include <unicode/ustring.h>  /* some more string fcns*/
#include <unicode/uchar.h>    /* char names           */

struct yaz_stemmer_t
{
    int implementation;
    union {
        struct sb_stemmer *snowballer;
    };
};

yaz_stemmer_p yaz_stemmer_create(const char *locale, const char *rule, UErrorCode *status) {
    return 0;
}

yaz_stemmer_p yaz_stemmer_clone(yaz_stemmer_p stemmer) {
    return 0;
}

void yaz_stemmer_stem(yaz_stemmer_p stemmer, struct icu_buf_utf16 *dst, struct icu_buf_utf16* src, UErrorCode *status) {

}

void yaz_stemmer_destroy(yaz_stemmer_p stemmer) {


}

#endif /* YAZ_HAVE_ICU */<commit_msg>First implementation of yaz_stemmer API. Can compile.<commit_after>

#if HAVE_CONFIG_H
#include "config.h"
#endif

#if YAZ_HAVE_ICU

#include <yaz/yconfig.h>

#include <yaz/stemmer.h>

#include <yaz/xmalloc.h>

#include <libstemmer.h>

#include <unicode/ustring.h>  /* some more string fcns*/
#include <unicode/uchar.h>    /* char names           */

enum stemmer_implementation {
    yaz_snowball
};
struct yaz_stemmer_t
{
    int implementation;
    // Required for cloning.
    const char *locale;
    const char *rule;
    union {
        struct sb_stemmer *sb_stemmer;
    };
};

const char* yaz_stemmer_lookup_charenc(const char *charenc) {
    return charenc;
}

const char* yaz_stemmer_lookup_algorithm(const char *locale, const char *rule) {
    return rule;
}

yaz_stemmer_p yaz_stemmer_snowball_create(const char *locale, const char *rule, UErrorCode *status) {
    const char *charenc = yaz_stemmer_lookup_charenc(locale);
    const char *algorithm = yaz_stemmer_lookup_algorithm(locale,rule);
    struct sb_stemmer *stemmer = sb_stemmer_new(algorithm, charenc);
    yaz_stemmer_p yaz_stemmer;
    if (stemmer == 0) {
        *status = U_ARGUMENT_TYPE_MISMATCH;
        return 0;
    }
    yaz_stemmer = xmalloc(sizeof(*yaz_stemmer));
    yaz_stemmer->implementation = yaz_snowball;
    yaz_stemmer->locale = xstrdup(locale);
    yaz_stemmer->rule = xstrdup(rule);
    yaz_stemmer->sb_stemmer = stemmer;

    return yaz_stemmer;
}

yaz_stemmer_p yaz_stemmer_create(const char *locale, const char *rule, UErrorCode *status) {
    *status = U_ZERO_ERROR;
    // dispatch logic required if more algorithms is implemented.
    return yaz_stemmer_snowball_create(locale, rule, status);
}

yaz_stemmer_p yaz_stemmer_clone(yaz_stemmer_p stemmer) {
    UErrorCode error = U_ZERO_ERROR;
    return yaz_stemmer_create(stemmer->locale, stemmer->rule, &error);
}

void yaz_stemmer_stem(yaz_stemmer_p stemmer, struct icu_buf_utf16 *dst, struct icu_buf_utf16* src, UErrorCode *status)
{
    switch(stemmer->implementation) {
    case yaz_snowball: {
        int length;
        struct icu_buf_utf8 *utf8_buf = icu_buf_utf8_create(0);
        icu_utf16_to_utf8(utf8_buf, src, status);
        if (*status == U_ZERO_ERROR) {
            const char *sb_symbol = sb_stemmer_stem(stemmer->sb_stemmer, icu_buf_utf8_to_cstr(utf8_buf), length);
            if (sb_symbol == 0) {
                icu_buf_utf16_copy(dst, src);
            }
            else {
                icu_utf16_from_utf8_cstr(dst, sb_symbol, status);
            }
        }
        return ;
        break;
    }
    }
}

void yaz_stemmer_destroy(yaz_stemmer_p stemmer) {
    switch (stemmer->implementation) {
    case yaz_snowball:
        sb_stemmer_delete(stemmer->sb_stemmer);
        break;
    }
    free(stemmer->locale);
    free(stemmer->rule);
    free(stemmer);
}

#endif /* YAZ_HAVE_ICU */
{'fraction_non_alphanumeric': 0.10157328181065416, 'fraction_numerical': 0.007728401876897599, 'mean_word_length': 3.474074074074074, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3044348', 'n_tokens_mistral': 1449, 'n_tokens_neox': 1355, 'n_words': 264}