<commit_before>/****************************************************************************
    
    unicode.hpp - Utility functions for encoding and decoding UTF-8 strings
    
    Copyright (C) 2006-2007 <NAME> <<EMAIL>>
    
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
    
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 01222-1307  USA

****************************************************************************/

#ifndef UNICODE_HPP
#define UNICODE_HPP

#include <iostream>
#include <string>
#include <utility>


using namespace std;
using namespace boost::spirit::classic;


template <typename ScannerT>
pair<unsigned int, unsigned int> decode_utf8(ScannerT const& scan) {
  
  typedef typename ScannerT::iterator_t iterator_t;
  typedef typename ScannerT::value_t value_t;
  
  if (scan.at_end())
    return make_pair(0, 0);
  
  iterator_t iter(scan.first);
  
  value_t ch = *scan.first;
  
  unsigned int size;
  unsigned int codepoint = 0;
  
  // one byte char
  if ((unsigned char)ch < 0x80) {
    size = 1;
    codepoint = ch;
  }
  
  // two byte char
  else if ((unsigned char)ch >= 192 && (unsigned char)ch < 224) {
    codepoint = ((unsigned char)ch && 0x1F) << 6;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += (unsigned char)(*iter) && 0x3F;
    size = 2;
  }
  
  // three byte char
  else if ((unsigned char)ch >= 224 && (unsigned char)ch < 240) {
    codepoint = ((unsigned char)ch && 0x0F) << 12;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) && 0x3F) << 6;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) && 0x3F);
    size = 3;
  }
  
  // four byte char
  else if ((unsigned char)ch >= 240 && (unsigned char)ch < 248) {
    codepoint = ((unsigned char)ch && 0x07) << 18;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) && 0x3F) << 12;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) && 0x3F) << 6;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) && 0x3F);
    size = 4;
  }
  
  // this is not allowed in Turtle!
  else {
    assert(!"Unexpected parse error");
  }
  
  return make_pair(size, codepoint);
}
<commit_msg>Use bitwise operators for bitwise comparisons<commit_after>/****************************************************************************
    
    unicode.hpp - Utility functions for encoding and decoding UTF-8 strings
    
    Copyright (C) 2006-2007 <NAME> <<EMAIL>>
    
    This program is free software; you can redistribute it and/or modify
    it under the terms of the GNU General Public License as published by
    the Free Software Foundation; either version 3 of the License, or
    (at your option) any later version.
    
    This program is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
    GNU General Public License for more details.
    
    You should have received a copy of the GNU General Public License
    along with this program; if not, write to the Free Software
    Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 01222-1307  USA

****************************************************************************/

#ifndef UNICODE_HPP
#define UNICODE_HPP

#include <iostream>
#include <string>
#include <utility>


using namespace std;
using namespace boost::spirit::classic;


template <typename ScannerT>
pair<unsigned int, unsigned int> decode_utf8(ScannerT const& scan) {
  
  typedef typename ScannerT::iterator_t iterator_t;
  typedef typename ScannerT::value_t value_t;
  
  if (scan.at_end())
    return make_pair(0, 0);
  
  iterator_t iter(scan.first);
  
  value_t ch = *scan.first;
  
  unsigned int size;
  unsigned int codepoint = 0;
  
  // one byte char
  if ((unsigned char)ch < 0x80) {
    size = 1;
    codepoint = ch;
  }
  
  // two byte char
  else if ((unsigned char)ch >= 192 && (unsigned char)ch < 224) {
    codepoint = ((unsigned char)ch & 0x1F) << 6;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += (unsigned char)(*iter) & 0x3F;
    size = 2;
  }
  
  // three byte char
  else if ((unsigned char)ch >= 224 && (unsigned char)ch < 240) {
    codepoint = ((unsigned char)ch & 0x0F) << 12;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) & 0x3F) << 6;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) & 0x3F);
    size = 3;
  }
  
  // four byte char
  else if ((unsigned char)ch >= 240 && (unsigned char)ch < 248) {
    codepoint = ((unsigned char)ch & 0x07) << 18;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) & 0x3F) << 12;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) & 0x3F) << 6;
    ++iter;
    if (iter == scan.last)
      return make_pair(0, 0);
    codepoint += ((unsigned char)(*iter) & 0x3F);
    size = 4;
  }
  
  // this is not allowed in Turtle!
  else {
    assert(!"Unexpected parse error");
  }
  
  return make_pair(size, codepoint);
}

{'fraction_non_alphanumeric': 0.16339005451842062, 'fraction_numerical': 0.030728564348257062, 'mean_word_length': 3.096075778078484, 'pattern_counts': {'":': 0, '<': 51, '<?xml version=': 0, '>': 25, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3274886', 'n_tokens_mistral': 2045, 'n_tokens_neox': 1887, 'n_words': 657}