<commit_before>/* This file is part of the YAZ toolkit.
 * Copyright (C) 1995-2013 Index Data
 * See the file LICENSE for details.
 */
#if HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdlib.h>
#include <errno.h>
#include <string.h>

#include <yaz/yaz-util.h>
#include <yaz/test.h>

#define ESC "\x1b"

static int compare_buffers(char *msg, int no,
                           int expect_len, const char *expect_buf,
                           int got_len, const char *got_buf)
{
    if (expect_len == got_len
        && !memcmp(expect_buf, got_buf, expect_len))
        return 1;

    if (0) /* use 1 see how the buffers differ (for debug purposes) */
    {
        int i;
        printf("tsticonv test=%s i=%d failed\n", msg, no);
        printf("off got exp\n");
        for (i = 0; i<got_len || i<expect_len; i++)
        {
            char got_char[10];
            char expect_char[10];

            if (i < got_len)
                sprintf(got_char, "%02X", got_buf[i]);
            else
                sprintf(got_char, "?  ");

            if (i < expect_len)
                sprintf(expect_char, "%02X", expect_buf[i]);
            else
                sprintf(expect_char, "?  ");

            printf("%02d  %s  %s %c\n",
                   i, got_char, expect_char, got_buf[i] == expect_buf[i] ?
                   ' ' : '*');

        }
    }
    return 0;
}

static int tst_convert_l(yaz_iconv_t cd, size_t in_len, const char *in_buf,
                         size_t expect_len, const char *expect_buf)
{
    size_t r;
    char *inbuf= (char*) in_buf;
    size_t inbytesleft = in_len > 0 ? in_len : strlen(in_buf);
    char outbuf0[64];
    char *outbuf = outbuf0;

    while (inbytesleft)
    {
        size_t outbytesleft = outbuf0 + sizeof(outbuf0) - outbuf;
        if (outbytesleft > 12)
            outbytesleft = 12;
        r = yaz_iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
        if (r == (size_t) (-1))
        {
            int e = yaz_iconv_error(cd);
            if (e != YAZ_ICONV_E2BIG)
                return 0;
        }
        else
        {
            yaz_iconv(cd, 0, 0, &outbuf, &outbytesleft);
            break;
        }
    }

    return compare_buffers("tsticonv 22", 0,
                           expect_len, expect_buf,
                           outbuf - outbuf0, outbuf0);
}

static int tst_convert_x(yaz_iconv_t cd, const char *buf, const char *cmpbuf,
                         int expect_error)
{
    int ret = 1;
    WRBUF b = wrbuf_alloc();
    char outbuf[16];
    size_t inbytesleft = strlen(buf);
    const char *inp = buf;
    int rounds = 0;
    for (rounds = 0; inbytesleft && rounds < (int) sizeof(outbuf); rounds++)
    {
        size_t outbytesleft = sizeof(outbuf);
        char *outp = outbuf;
        size_t r = yaz_iconv(cd, (char**) &inp,  &inbytesleft,
                             &outp, &outbytesleft);
        wrbuf_write(b, outbuf, outp - outbuf);
        if (r == (size_t) (-1))
        {
            int e = yaz_iconv_error(cd);
            if (e != YAZ_ICONV_E2BIG)
            {
                if (expect_error != -1)
                    if (e != expect_error)
                        ret = 0;
                break;
            }
        }
        else
        {
            size_t outbytesleft = sizeof(outbuf);
            char *outp = outbuf;
            r = yaz_iconv(cd, 0, 0, &outp, &outbytesleft);
            wrbuf_write(b, outbuf, outp - outbuf);
            if (expect_error != -1)
                if (expect_error)
                    ret = 0;
            break;
        }
    }
    if (wrbuf_len(b) == strlen(cmpbuf)
        && !memcmp(cmpbuf, wrbuf_buf(b), wrbuf_len(b)))
        ;
    else
    {
        WRBUF w = wrbuf_alloc();

        ret = 0;
        wrbuf_rewind(w);
        wrbuf_puts_escaped(w, buf);
        yaz_log(YLOG_LOG, "input %s", wrbuf_cstr(w));

        wrbuf_rewind(w);
        wrbuf_write_escaped(w, wrbuf_buf(b), wrbuf_len(b));
        yaz_log(YLOG_LOG, "got %s", wrbuf_cstr(w));

        wrbuf_rewind(w);
        wrbuf_puts_escaped(w, cmpbuf);
        yaz_log(YLOG_LOG, "exp %s", wrbuf_cstr(w));

        wrbuf_destroy(w);
    }

    wrbuf_destroy(b);
    return ret;
}

static int tst_convert(yaz_iconv_t cd, const char *buf, const char *cmpbuf)
{
    return tst_convert_x(cd, buf, cmpbuf, 0);
}

/* some test strings in ISO-8859-1 format */
static const char *iso_8859_1_a[] = {
    "ax" ,
    "\xd8",
    "eneb\346r",
    "\xe5" "\xd8",
    "\xe5" "\xd8" "b",
    "\xe5" "\xe5",
    0 };

static void tst_marc8_to_ucs4b(void)
{
    yaz_iconv_t cd = yaz_iconv_open("UCS4", "MARC8");
    YAZ_CHECK(cd);
    if (!cd)
        return;

    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\033$1" "\x21\x2B\x3B" /* FF1F */ "\033(B" "o",
                  8,
                  "\x00\x00\xFF\x1F" "\x00\x00\x00o"));
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\033$1" "\x6F\x77\x29" /* AE0E */
                  "\x6F\x52\x7C" /* c0F4 */ "\033(B",
                  8,
                  "\x00\x00\xAE\x0E" "\x00\x00\xC0\xF4"));
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\033$1"
                  "\x21\x50\x6E"  /* UCS 7CFB */
                  "\x21\x51\x31"  /* UCS 7D71 */
                  "\x21\x3A\x67"  /* UCS 5B89 */
                  "\x21\x33\x22"  /* UCS 5168 */
                  "\x21\x33\x53"  /* UCS 5206 */
                  "\x21\x44\x2B"  /* UCS 6790 */
                  "\033(B",
                  24,
                  "\x00\x00\x7C\xFB"
                  "\x00\x00\x7D\x71"
                  "\x00\x00\x5B\x89"
                  "\x00\x00\x51\x68"
                  "\x00\x00\x52\x06"
                  "\x00\x00\x67\x90"));

    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xB0\xB2",     /* AYN and oSLASH */
                  8,
                  "\x00\x00\x02\xBB"  "\x00\x00\x00\xF8"));
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xF6\x61",     /* a underscore */
                  8,
                  "\x00\x00\x00\x61"  "\x00\x00\x03\x32"));

    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\x61\xC2",     /* a, phonorecord mark */
                  8,
                  "\x00\x00\x00\x61"  "\x00\x00\x21\x17"));

    /* bug #258 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "el" "\xe8" "am\xe8" "an", /* elaman where a is a" */
                  32,
                  "\x00\x00\x00" "e"
                  "\x00\x00\x00" "l"
                  "\x00\x00\x00" "a"
                  "\x00\x00\x03\x08"
                  "\x00\x00\x00" "m"
                  "\x00\x00\x00" "a"
                  "\x00\x00\x03\x08"
                  "\x00\x00\x00" "n"));
    /* bug #260 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xe5\xe8\x41",
                  12,
                  "\x00\x00\x00\x41" "\x00\x00\x03\x04" "\x00\x00\x03\x08"));
    /* bug #416 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xEB\x74\xEC\x73",
                  12,
                  "\x00\x00\x00\x74" "\x00\x00\x03\x61" "\x00\x00\x00\x73"));
    /* bug #416 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xFA\x74\xFB\x73",
                  12,
                  "\x00\x00\x00\x74" "\x00\x00\x03\x60" "\x00\x00\x00\x73"));

    yaz_iconv_close(cd);
}

static void tst_ucs4b_to_utf8(void)
{
    yaz_iconv_t cd = yaz_iconv_open("UTF8", "UCS4");
    YAZ_CHECK(cd);
    if (!cd)
        return;
    YAZ_CHECK(tst_convert_l(
                  cd,
                  8,
                  "\x00\x00\xFF\x1F\x00\x00\x00o",
                  4,
                  "\xEF\xBC\x9F\x6F"));

    YAZ_CHECK(tst_convert_l(
                  cd,
                  8,
                  "\x00\x00\xAE\x0E\x00\x00\xC0\xF4",
                  6,
                  "\xEA\xB8\x8E\xEC\x83\xB4"));
    yaz_iconv_close(cd);
}

static void dconvert(int mandatory, const char *tmpcode)
{
    int i;
    int ret;<commit_msg>test_iconv: make variable local<commit_after>/* This file is part of the YAZ toolkit.
 * Copyright (C) 1995-2013 Index Data
 * See the file LICENSE for details.
 */
#if HAVE_CONFIG_H
#include <config.h>
#endif

#include <stdlib.h>
#include <errno.h>
#include <string.h>

#include <yaz/yaz-util.h>
#include <yaz/test.h>

#define ESC "\x1b"

static int compare_buffers(char *msg, int no,
                           int expect_len, const char *expect_buf,
                           int got_len, const char *got_buf)
{
    if (expect_len == got_len
        && !memcmp(expect_buf, got_buf, expect_len))
        return 1;

    if (0) /* use 1 see how the buffers differ (for debug purposes) */
    {
        int i;
        printf("tsticonv test=%s i=%d failed\n", msg, no);
        printf("off got exp\n");
        for (i = 0; i<got_len || i<expect_len; i++)
        {
            char got_char[10];
            char expect_char[10];

            if (i < got_len)
                sprintf(got_char, "%02X", got_buf[i]);
            else
                sprintf(got_char, "?  ");

            if (i < expect_len)
                sprintf(expect_char, "%02X", expect_buf[i]);
            else
                sprintf(expect_char, "?  ");

            printf("%02d  %s  %s %c\n",
                   i, got_char, expect_char, got_buf[i] == expect_buf[i] ?
                   ' ' : '*');

        }
    }
    return 0;
}

static int tst_convert_l(yaz_iconv_t cd, size_t in_len, const char *in_buf,
                         size_t expect_len, const char *expect_buf)
{
    size_t r;
    char *inbuf= (char*) in_buf;
    size_t inbytesleft = in_len > 0 ? in_len : strlen(in_buf);
    char outbuf0[64];
    char *outbuf = outbuf0;

    while (inbytesleft)
    {
        size_t outbytesleft = outbuf0 + sizeof(outbuf0) - outbuf;
        if (outbytesleft > 12)
            outbytesleft = 12;
        r = yaz_iconv(cd, &inbuf, &inbytesleft, &outbuf, &outbytesleft);
        if (r == (size_t) (-1))
        {
            int e = yaz_iconv_error(cd);
            if (e != YAZ_ICONV_E2BIG)
                return 0;
        }
        else
        {
            yaz_iconv(cd, 0, 0, &outbuf, &outbytesleft);
            break;
        }
    }

    return compare_buffers("tsticonv 22", 0,
                           expect_len, expect_buf,
                           outbuf - outbuf0, outbuf0);
}

static int tst_convert_x(yaz_iconv_t cd, const char *buf, const char *cmpbuf,
                         int expect_error)
{
    int ret = 1;
    WRBUF b = wrbuf_alloc();
    char outbuf[16];
    size_t inbytesleft = strlen(buf);
    const char *inp = buf;
    int rounds = 0;
    for (rounds = 0; inbytesleft && rounds < (int) sizeof(outbuf); rounds++)
    {
        size_t outbytesleft = sizeof(outbuf);
        char *outp = outbuf;
        size_t r = yaz_iconv(cd, (char**) &inp,  &inbytesleft,
                             &outp, &outbytesleft);
        wrbuf_write(b, outbuf, outp - outbuf);
        if (r == (size_t) (-1))
        {
            int e = yaz_iconv_error(cd);
            if (e != YAZ_ICONV_E2BIG)
            {
                if (expect_error != -1)
                    if (e != expect_error)
                        ret = 0;
                break;
            }
        }
        else
        {
            size_t outbytesleft = sizeof(outbuf);
            char *outp = outbuf;
            r = yaz_iconv(cd, 0, 0, &outp, &outbytesleft);
            wrbuf_write(b, outbuf, outp - outbuf);
            if (expect_error != -1)
                if (expect_error)
                    ret = 0;
            break;
        }
    }
    if (wrbuf_len(b) == strlen(cmpbuf)
        && !memcmp(cmpbuf, wrbuf_buf(b), wrbuf_len(b)))
        ;
    else
    {
        WRBUF w = wrbuf_alloc();

        ret = 0;
        wrbuf_rewind(w);
        wrbuf_puts_escaped(w, buf);
        yaz_log(YLOG_LOG, "input %s", wrbuf_cstr(w));

        wrbuf_rewind(w);
        wrbuf_write_escaped(w, wrbuf_buf(b), wrbuf_len(b));
        yaz_log(YLOG_LOG, "got %s", wrbuf_cstr(w));

        wrbuf_rewind(w);
        wrbuf_puts_escaped(w, cmpbuf);
        yaz_log(YLOG_LOG, "exp %s", wrbuf_cstr(w));

        wrbuf_destroy(w);
    }

    wrbuf_destroy(b);
    return ret;
}

static int tst_convert(yaz_iconv_t cd, const char *buf, const char *cmpbuf)
{
    return tst_convert_x(cd, buf, cmpbuf, 0);
}

static void tst_marc8_to_ucs4b(void)
{
    yaz_iconv_t cd = yaz_iconv_open("UCS4", "MARC8");
    YAZ_CHECK(cd);
    if (!cd)
        return;

    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\033$1" "\x21\x2B\x3B" /* FF1F */ "\033(B" "o",
                  8,
                  "\x00\x00\xFF\x1F" "\x00\x00\x00o"));
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\033$1" "\x6F\x77\x29" /* AE0E */
                  "\x6F\x52\x7C" /* c0F4 */ "\033(B",
                  8,
                  "\x00\x00\xAE\x0E" "\x00\x00\xC0\xF4"));
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\033$1"
                  "\x21\x50\x6E"  /* UCS 7CFB */
                  "\x21\x51\x31"  /* UCS 7D71 */
                  "\x21\x3A\x67"  /* UCS 5B89 */
                  "\x21\x33\x22"  /* UCS 5168 */
                  "\x21\x33\x53"  /* UCS 5206 */
                  "\x21\x44\x2B"  /* UCS 6790 */
                  "\033(B",
                  24,
                  "\x00\x00\x7C\xFB"
                  "\x00\x00\x7D\x71"
                  "\x00\x00\x5B\x89"
                  "\x00\x00\x51\x68"
                  "\x00\x00\x52\x06"
                  "\x00\x00\x67\x90"));

    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xB0\xB2",     /* AYN and oSLASH */
                  8,
                  "\x00\x00\x02\xBB"  "\x00\x00\x00\xF8"));
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xF6\x61",     /* a underscore */
                  8,
                  "\x00\x00\x00\x61"  "\x00\x00\x03\x32"));

    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\x61\xC2",     /* a, phonorecord mark */
                  8,
                  "\x00\x00\x00\x61"  "\x00\x00\x21\x17"));

    /* bug #258 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "el" "\xe8" "am\xe8" "an", /* elaman where a is a" */
                  32,
                  "\x00\x00\x00" "e"
                  "\x00\x00\x00" "l"
                  "\x00\x00\x00" "a"
                  "\x00\x00\x03\x08"
                  "\x00\x00\x00" "m"
                  "\x00\x00\x00" "a"
                  "\x00\x00\x03\x08"
                  "\x00\x00\x00" "n"));
    /* bug #260 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xe5\xe8\x41",
                  12,
                  "\x00\x00\x00\x41" "\x00\x00\x03\x04" "\x00\x00\x03\x08"));
    /* bug #416 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xEB\x74\xEC\x73",
                  12,
                  "\x00\x00\x00\x74" "\x00\x00\x03\x61" "\x00\x00\x00\x73"));
    /* bug #416 */
    YAZ_CHECK(tst_convert_l(
                  cd,
                  0,
                  "\xFA\x74\xFB\x73",
                  12,
                  "\x00\x00\x00\x74" "\x00\x00\x03\x60" "\x00\x00\x00\x73"));

    yaz_iconv_close(cd);
}

static void tst_ucs4b_to_utf8(void)
{
    yaz_iconv_t cd = yaz_iconv_open("UTF8", "UCS4");
    YAZ_CHECK(cd);
    if (!cd)
        return;
    YAZ_CHECK(tst_convert_l(
                  cd,
                  8,
                  "\x00\x00\xFF\x1F\x00\x00\x00o",
                  4,
                  "\xEF\xBC\x9F\x6F"));

    YAZ_CHECK(tst_convert_l(
                  cd,
                  8,
                  "\x00\x00\xAE\x0E\x00\x00\xC0\xF4",
                  6,
                  "\xEA\xB8\x8E\xEC\x83\xB4"));
    yaz_iconv_close(cd);
}

static void dconvert(int mandatory, const char *tmpcode)
{
    /* some test strings in ISO-8859-1 format */
    static const char *iso_8859_1_a[] = {
        "ax" ,
        "\xd8",
        "eneb\346r",
        "\xe5" "\xd8",
        "\xe5" "\xd8" "b",
        "\xe5" "\xe5",
        0 };
    int i;
    int ret;
{'fraction_non_alphanumeric': 0.1397603420244475, 'fraction_numerical': 0.06081772746432228, 'mean_word_length': 1.2023604296512398, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 19, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '4778535', 'n_tokens_mistral': 6827, 'n_tokens_neox': 6083, 'n_words': 1124}