<commit_before>/* bzflag
 * Copyright (c) 1993 - 2008 <NAME>
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named COPYING that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

#include "bzUnicode.h"


// UTF8StringItr
UTF8StringItr& UTF8StringItr::operator ++()
{
  curPos = nextPos;

  unsigned int ch = 0;
  unsigned int extraBytesToRead = utf8bytes[(unsigned char)(*nextPos)];
  // falls through
  switch (extraBytesToRead)
  {
    case 6: ch += *nextPos++; ch <<= 6; /* remember, illegal UTF-8 */
    case 5: ch += *nextPos++; ch <<= 6; /* remember, illegal UTF-8 */
    case 4: ch += *nextPos++; ch <<= 6;
    case 3: ch += *nextPos++; ch <<= 6;
    case 2: ch += *nextPos++; ch <<= 6;
    case 1: ch += *nextPos++;
  }
  ch -= offsetsFromUTF8[extraBytesToRead-1];
  curChar = ch;

  return *this;
}

/* The first character in a UTF8 sequence indicates how many bytes
 * to read (among other things) */
const char UTF8StringItr::utf8bytes[256] = {
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
  3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4,5,5,5,5,6,6,6,6
};

/* Magic values subtracted from a buffer value during UTF8 conversion.
 * This table contains as many values as there might be trailing bytes
 * in a UTF-8 sequence. */
const unsigned long UTF8StringItr::offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
  0x03C82080UL, 0xFA082080UL, 0x82082080UL };


// bzUTF8Char

bzUTF8Char::bzUTF8Char(unsigned int ch)
{
  int bytesOut = 0;<commit_msg>don't use signed chars internally, even if you have to for the interface.  screws up bitshift followed by addition.<commit_after>/* bzflag
 * Copyright (c) 1993 - 2008 <NAME>
 *
 * This package is free software;  you can redistribute it and/or
 * modify it under the terms of the license found in the file
 * named COPYING that should have accompanied this file.
 *
 * THIS PACKAGE IS PROVIDED ``AS IS'' AND WITHOUT ANY EXPRESS OR
 * IMPLIED WARRANTIES, INCLUDING, WITHOUT LIMITATION, THE IMPLIED
 * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE.
 */

#include "bzUnicode.h"


// UTF8StringItr
UTF8StringItr& UTF8StringItr::operator ++()
{
  curPos = nextPos;

  // get rid of signedness before doing bit magic
  const unsigned char* np = reinterpret_cast<const unsigned char*>(nextPos);

  unsigned int ch = 0;
  unsigned int extraBytesToRead = utf8bytes[*np];
  // falls through
  switch (extraBytesToRead)
  {
    case 6: ch += *np++; ch <<= 6; /* remember, illegal UTF-8 */
    case 5: ch += *np++; ch <<= 6; /* remember, illegal UTF-8 */
    case 4: ch += *np++; ch <<= 6;
    case 3: ch += *np++; ch <<= 6;
    case 2: ch += *np++; ch <<= 6;
    case 1: ch += *np++;
  }

  // put the pointer back
  nextPos = reinterpret_cast<const char*>(np);
  ch -= offsetsFromUTF8[extraBytesToRead-1];
  curChar = ch;

  return *this;
}

/* The first character in a UTF8 sequence indicates how many bytes
 * to read (among other things) */
const char UTF8StringItr::utf8bytes[256] = {
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1, 1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,
  2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2, 2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,2,
  3,3,3,3,3,3,3,3,3,3,3,3,3,3,3,3, 4,4,4,4,4,4,4,4,5,5,5,5,6,6,6,6
};

/* Magic values subtracted from a buffer value during UTF8 conversion.
 * This table contains as many values as there might be trailing bytes
 * in a UTF-8 sequence. */
const unsigned long UTF8StringItr::offsetsFromUTF8[6] = { 0x00000000UL, 0x00003080UL, 0x000E2080UL,
  0x03C82080UL, 0xFA082080UL, 0x82082080UL };


// bzUTF8Char

bzUTF8Char::bzUTF8Char(unsigned int ch)
{
  int bytesOut = 0;
{'fraction_non_alphanumeric': 0.19900605012964565, 'fraction_numerical': 0.1508210890233362, 'mean_word_length': 4.6797546012269935, 'pattern_counts': {'":': 0, '<': 27, '<?xml version=': 0, '>': 7, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 2, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5969196', 'n_tokens_mistral': 2349, 'n_tokens_neox': 2178, 'n_words': 454}