<commit_before>#include "evas_common.h"
#include "evas_encoding.h"

EAPI Eina_Unicode
evas_common_encoding_utf8_get_next(const char *buf, int *iindex)
{
   /* Reads UTF8 bytes from @buf, starting at *@index and returns
    * the decoded code point at iindex offset, and advances iindex
    * to the next code point after this.
    *
    * Returns 0 to indicate there is no next char
    */
   int index = *iindex, len, r;
   unsigned char d, d2, d3, d4;

   /* if this char is the null terminator, exit */
   if (!buf[index])
     return 0;
     
   d = buf[index++];

   while (buf[index] && ((buf[index] & 0xc0) == 0x80))
     index++;
   len = index - *iindex;

   if (len == 1)
      r = d;
   else if (len == 2)
     {
	/* 2 bytes */
        d2 = buf[*iindex + 1];
	r = d & 0x1f; /* copy lower 5 */
	r <<= 6;
	r |= (d2 & 0x3f); /* copy lower 6 */
     }
   else if (len == 3)
     {
	/* 3 bytes */
        d2 = buf[*iindex + 1];
        d3 = buf[*iindex + 2];
	r = d & 0x0f; /* copy lower 4 */
	r <<= 6;
	r |= (d2 & 0x3f);
	r <<= 6;
	r |= (d3 & 0x3f);
     }
   else
     {
	/* 4 bytes */
        d2 = buf[*iindex + 1];
        d3 = buf[*iindex + 2];
        d4 = buf[*iindex + 3];
	r = d & 0x0f; /* copy lower 4 */
	r <<= 6;
	r |= (d2 & 0x3f);
	r <<= 6;
	r |= (d3 & 0x3f);
	r <<= 6;
	r |= (d4 & 0x3f);
     }

   *iindex = index;
   return r;
}

EAPI Eina_Unicode
evas_common_encoding_utf8_get_prev(const char *buf, int *iindex)
{<commit_msg>more robust utf8 walking code to survive malformed utf8 and not crash in the next ut8 char get code.<commit_after>#include "evas_common.h"
#include "evas_encoding.h"

EAPI Eina_Unicode
evas_common_encoding_utf8_get_next(const char *buf, int *iindex)
{
   /* Reads UTF8 bytes from @buf, starting at *@index and returns
    * the decoded code point at iindex offset, and advances iindex
    * to the next code point after this.
    *
    * Returns 0 to indicate there is no next char
    */
#if 1
   int index = *iindex, r;
   unsigned char d;

   /* if this char is the null terminator, exit */
   d = buf[index];
   if (!d) return 0;
   index++;
     
   if ((d & 0x80) == 0) // 1 byte ascii (7bit) - 0xxxxxxx
     {
        *iindex = index;
        return d;
     }
   if ((d & 0xe0) == 0xc0) // 2 byte utf8 (11bit) - 110xxxxx 10xxxxxx
     {
        r = (d & 0x1f) << 6;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f);
        *iindex = index;
        return r;
     }
   if ((d & 0xf0) == 0xe0) // 3 byte utf8 (16bit) - 1110xxxx 10xxxxxx 10xxxxxx
     {
        r = (d & 0x0f) << 12;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 6;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f);
        *iindex = index;
        return r;
     }
   if ((d & 0xf8) == 0xf0) // 4 byte utf8 (21bit) - 11110xxx 10xxxxxx 10xxxxxx 10xxxxxx
     {
        r = (d & 0x07) << 18;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 12;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 6;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f);
        *iindex = index;
        return r;
     }
   if ((d & 0xfc) == 0xf8) // 5 byte utf8 (26bit) - 111110xx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
     {
        r = (d & 0x03) << 24;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 18;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 12;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 6;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f);
        *iindex = index;
        return r;
     }
   if ((d & 0xfe) == 0xfc) // 6 byte utf8 (31bit) - 1111110x 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx 10xxxxxx
     {
        r = (d & 0x01) << 30;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 24;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 18;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 12;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f) << 6;
        d = buf[index];
        if (!d) return 0;
        index++;
        r |= (d & 0x3f);
        *iindex = index;
        return r;
     }
   return 0;
#else
   int index = *iindex, r, len;
   unsigned char d, d2, d3, d4;
   
   /* if this char is the null terminator, exit */
   if (!buf[index]) return 0;
     
   d = buf[index++];

   while (buf[index] && ((buf[index] & 0xc0) == 0x80))
     index++;
   len = index - *iindex;

   if (len == 1)
      r = d;
   else if (len == 2)
     {
	/* 2 bytes */
        d2 = buf[*iindex + 1];
        if (d2 == 0)
          {
             *iindex = *iindex + 1;
             return 0;
          }
	r = d & 0x1f; /* copy lower 5 */
	r <<= 6;
	r |= (d2 & 0x3f); /* copy lower 6 */
     }
   else if (len == 3)
     {
	/* 3 bytes */
        d2 = buf[*iindex + 1];
        if (d2 == 0)
          {
             *iindex = *iindex + 1;
             return 0;
          }
        d3 = buf[*iindex + 2];
        if (d3 == 0)
          {
             *iindex = *iindex + 2;
             return 0;
          }
	r = d & 0x0f; /* copy lower 4 */
	r <<= 6;
	r |= (d2 & 0x3f);
	r <<= 6;
	r |= (d3 & 0x3f);
     }
   else
     {
	/* 4 bytes */
        d2 = buf[*iindex + 1];
        if (d2 == 0)
          {
             *iindex = *iindex + 1;
             return 0;
          }
        d3 = buf[*iindex + 2];
        if (d3 == 0)
          {
             *iindex = *iindex + 2;
             return 0;
          }
        d4 = buf[*iindex + 3];
        if (d4 == 0)
          {
             *iindex = *iindex + 3;
             return 0;
          }
	r = d & 0x0f; /* copy lower 4 */
	r <<= 6;
	r |= (d2 & 0x3f);
	r <<= 6;
	r |= (d3 & 0x3f);
	r <<= 6;
	r |= (d4 & 0x3f);
     }
   *iindex = index;
   return r;
#endif
}

EAPI Eina_Unicode
evas_common_encoding_utf8_get_prev(const char *buf, int *iindex)
{
{'fraction_non_alphanumeric': 0.1611450627211322, 'fraction_numerical': 0.055001608234158894, 'mean_word_length': 1.4757165605095541, 'pattern_counts': {'":': 0, '<': 57, '<?xml version=': 0, '>': 3, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 1, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '1772712', 'n_tokens_mistral': 2674, 'n_tokens_neox': 2406, 'n_words': 729}