<commit_before>#ifndef SASS_UTF8_STRING
#define SASS_UTF8_STRING

#include <string>

namespace Sass {
  namespace UTF_8 {
    using std::string;
    // class utf8_string {
    //   string s_;
    // public:
    //   utf8_string(const string &s): s_(s) {}
    //   utf8_string(const char* c): s_(string(c)) {}

    //   char operator[](size_t i);
    //   size_t length();
    //   size_t byte_to_char(size_t i);
    // };

    // function that will count the number of code points (utf-8 characters) from the given beginning to the given end
    size_t code_point_count(const string& str, size_t start, size_t end) {
      size_t len = 0;
      size_t i = start;

      while (i < end) {
        unsigned char c = static_cast<unsigned char>(str[i]);
        if (c < 128) {
          // it's a single-byte character
          ++len;
          ++i;
        }
        // it's a multi bit sequence and presumably it's a leading bit
        else {
          ++i; // go to the next byte
          // see if it's still part of the sequence
          while ((i < end) && ((static_cast<unsigned char>(str[i]) & 0b11000000) == 0b10000000)) {
            ++i;
          }
          // when it's not [aka a new leading bit], increment and move on
          ++len;
        }
      }
      return len;
    }

    size_t code_point_count(const string& str) {
      return code_point_count(str, 0, str.length());
    }

    // function that will return the byte offset of a code point in a
    size_t code_point_offset_to_byte_offset(const string& str, size_t offset) {
      size_t i = 0;
      size_t len = 0;

      while (len < offset) {
        unsigned char c = static_cast<unsigned char>(str[i]);
        if (c < 128) {
          // it's a single-byte character
          ++len;
          ++i;
        }
        // it's a multi bit sequence and presumably it's a leading bit
        else {
          ++i; // go to the next byte
          // see if it's still part of the sequence
          while ((i < str.length()) && ((static_cast<unsigned char>(str[i]) & 0b11000000) == 0b10000000)) {
            ++i;
          }
          // when it's not [aka a new leading bit], increment and move on
          ++len;
        }
      }
      return i;
    }

    // function that returns number of bytes in a character in a string
    size_t length_of_code_point_at(const string& str, size_t pos) {
      unsigned char c = static_cast<unsigned char>(str[pos]);
      size_t i = 0;
      if(c < 128) {
        return 1;
      } else {
        ++i; // go to the next byte
        ++pos;
        // see if it's still part of the sequence
        while ((i < str.length()) && ((static_cast<unsigned char>(str[pos]) & 0b11000000) == 0b10000000)) {
          ++i;
          ++pos;
        }
      }
      return i;
    }

    // function that will return a normalized index, given a crazy one
    size_t normalize_index(int index, size_t len) {
      int signed_len = len;
      // assuming the index is 1-based
      // we are returning a 0-based index
      if (index > 0 && index <= signed_len) {
        // positive and within string length
        return index-1;
      } 
      else if (index > signed_len) {
        // positive and past string length
        return len;
      }
      else if (index == 0) {
        return 0;
      }
      else if (std::abs(index) <= signed_len) {
        // negative and within string length
        return index + signed_len;
      }
      else {
        // negative and past string length
        return 0;
      }
    }

  }
}

#endif <commit_msg>including a necessary header file<commit_after>#ifndef SASS_UTF8_STRING
#define SASS_UTF8_STRING

#include <string>
#include <cmath>

namespace Sass {
  namespace UTF_8 {
    using std::string;
    // class utf8_string {
    //   string s_;
    // public:
    //   utf8_string(const string &s): s_(s) {}
    //   utf8_string(const char* c): s_(string(c)) {}

    //   char operator[](size_t i);
    //   size_t length();
    //   size_t byte_to_char(size_t i);
    // };

    // function that will count the number of code points (utf-8 characters) from the given beginning to the given end
    size_t code_point_count(const string& str, size_t start, size_t end) {
      size_t len = 0;
      size_t i = start;

      while (i < end) {
        unsigned char c = static_cast<unsigned char>(str[i]);
        if (c < 128) {
          // it's a single-byte character
          ++len;
          ++i;
        }
        // it's a multi bit sequence and presumably it's a leading bit
        else {
          ++i; // go to the next byte
          // see if it's still part of the sequence
          while ((i < end) && ((static_cast<unsigned char>(str[i]) & 0b11000000) == 0b10000000)) {
            ++i;
          }
          // when it's not [aka a new leading bit], increment and move on
          ++len;
        }
      }
      return len;
    }

    size_t code_point_count(const string& str) {
      return code_point_count(str, 0, str.length());
    }

    // function that will return the byte offset of a code point in a
    size_t code_point_offset_to_byte_offset(const string& str, size_t offset) {
      size_t i = 0;
      size_t len = 0;

      while (len < offset) {
        unsigned char c = static_cast<unsigned char>(str[i]);
        if (c < 128) {
          // it's a single-byte character
          ++len;
          ++i;
        }
        // it's a multi bit sequence and presumably it's a leading bit
        else {
          ++i; // go to the next byte
          // see if it's still part of the sequence
          while ((i < str.length()) && ((static_cast<unsigned char>(str[i]) & 0b11000000) == 0b10000000)) {
            ++i;
          }
          // when it's not [aka a new leading bit], increment and move on
          ++len;
        }
      }
      return i;
    }

    // function that returns number of bytes in a character in a string
    size_t length_of_code_point_at(const string& str, size_t pos) {
      unsigned char c = static_cast<unsigned char>(str[pos]);
      size_t i = 0;
      if(c < 128) {
        return 1;
      } else {
        ++i; // go to the next byte
        ++pos;
        // see if it's still part of the sequence
        while ((i < str.length()) && ((static_cast<unsigned char>(str[pos]) & 0b11000000) == 0b10000000)) {
          ++i;
          ++pos;
        }
      }
      return i;
    }

    // function that will return a normalized index, given a crazy one
    size_t normalize_index(int index, size_t len) {
      int signed_len = len;
      // assuming the index is 1-based
      // we are returning a 0-based index
      if (index > 0 && index <= signed_len) {
        // positive and within string length
        return index-1;
      } 
      else if (index > signed_len) {
        // positive and past string length
        return len;
      }
      else if (index == 0) {
        return 0;
      }
      else if (std::abs(index) <= signed_len) {
        // negative and within string length
        return index + signed_len;
      }
      else {
        // negative and past string length
        return 0;
      }
    }

  }
}

#endif 
{'fraction_non_alphanumeric': 0.11142100125505508, 'fraction_numerical': 0.023148793752614697, 'mean_word_length': 1.755282366500192, 'pattern_counts': {'":': 0, '<': 38, '<?xml version=': 0, '>': 22, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 6, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6911063', 'n_tokens_mistral': 2337, 'n_tokens_neox': 2164, 'n_words': 742}