<commit_before>/// \file json.cpp Holds all JSON-related code.

#include "json.h"
#include "bitfields.h"
#include "defines.h"
#include <arpa/inet.h> //for htonl
#include <fstream>
#include <sstream>
#include <stdint.h> //for uint64_t
#include <stdlib.h>
#include <string.h> //for memcpy

/// Construct from a root Value to iterate over.
JSON::Iter::Iter(Value &root){
  myType = root.myType;
  i = 0;
  r = &root;
  if (!root.size()){myType = JSON::EMPTY;}
  if (myType == JSON::ARRAY){aIt = root.arrVal.begin();}
  if (myType == JSON::OBJECT){oIt = root.objVal.begin();}
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
JSON::Value &JSON::Iter::operator*() const{
  if (myType == JSON::ARRAY && aIt != r->arrVal.end()){return **aIt;}
  if (myType == JSON::OBJECT && oIt != r->objVal.end()){return *(oIt->second);}
  static JSON::Value error;
  WARN_MSG("Dereferenced invalid JSON iterator");
  return error;
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
JSON::Value *JSON::Iter::operator->() const{
  return &(operator*());
}

/// True if not done iterating.
JSON::Iter::operator bool() const{
  return ((myType == JSON::ARRAY && aIt != r->arrVal.end()) ||
          (myType == JSON::OBJECT && oIt != r->objVal.end()));
}

/// Go to next iteration.
JSON::Iter &JSON::Iter::operator++(){
  if (*this){
    ++i;
    if (myType == JSON::ARRAY){++aIt;}
    if (myType == JSON::OBJECT){++oIt;}
  }
  return *this;
}

/// Return the name of the current indice.
const std::string &JSON::Iter::key() const{
  if (myType == JSON::OBJECT && *this){return oIt->first;}
  static const std::string empty;
  WARN_MSG("Got key from invalid JSON iterator");
  return empty;
}

/// Return the number of the current indice.
uint32_t JSON::Iter::num() const{
  return i;
}

/// Delete the current indice from the parent JSON::Value.
void JSON::Iter::remove(){
  if (*this){
    if (myType == JSON::ARRAY){
      r->removeMember(aIt);
      return;
    }
    if (myType == JSON::OBJECT){
      r->removeMember(oIt);
      return;
    }
  }
}

/// Construct from a root Value to iterate over.
JSON::ConstIter::ConstIter(const Value &root){
  myType = root.myType;
  i = 0;
  r = &root;
  if (!root.size()){myType = JSON::EMPTY;}
  if (myType == JSON::ARRAY){aIt = root.arrVal.begin();}
  if (myType == JSON::OBJECT){oIt = root.objVal.begin();}
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
const JSON::Value &JSON::ConstIter::operator*() const{
  if (myType == JSON::ARRAY && aIt != r->arrVal.end()){return **aIt;}
  if (myType == JSON::OBJECT && oIt != r->objVal.end()){return *(oIt->second);}
  static JSON::Value error;
  WARN_MSG("Dereferenced invalid JSON iterator");
  return error;
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
const JSON::Value *JSON::ConstIter::operator->() const{
  return &(operator*());
}

/// True if not done iterating.
JSON::ConstIter::operator bool() const{
  return ((myType == JSON::ARRAY && aIt != r->arrVal.end()) ||
          (myType == JSON::OBJECT && oIt != r->objVal.end()));
}

/// Go to next iteration.
JSON::ConstIter &JSON::ConstIter::operator++(){
  if (*this){
    ++i;
    if (myType == JSON::ARRAY){++aIt;}
    if (myType == JSON::OBJECT){++oIt;}
  }
  return *this;
}

/// Return the name of the current indice.
const std::string &JSON::ConstIter::key() const{
  if (myType == JSON::OBJECT && *this){return oIt->first;}
  static const std::string empty;
  WARN_MSG("Got key from invalid JSON iterator");
  return empty;
}

/// Return the number of the current indice.
uint32_t JSON::ConstIter::num() const{
  return i;
}

static inline char c2hex(char c){
  if (c >= '0' && c <= '9') return c - '0';
  if (c >= 'a' && c <= 'f') return c - 'a' + 10;
  if (c >= 'A' && c <= 'F') return c - 'A' + 10;
  return 0;
}

static inline char hex2c(char c){
  if (c < 10){return '0' + c;}
  if (c < 16){return 'A' + (c - 10);}
  return '0';
}

static std::string UTF8(uint32_t c){
  std::string r;
  if (c <= 0x7F){
    r.append(1, c);
    return r;
  }
  if (c <= 0x7FF){
    r.append(1, 0xC0 | (c >> 6));
    r.append(1, 0x80 | (c & 0x3F));
    return r;
  }
  if (c <= 0x7FF){
    r.append(1, 0xC0 | (c >> 12));
    r.append(1, 0x80 | ((c >> 6) & 0x3F));
    r.append(1, 0x80 | (c & 0x3F));
    return r;
  }
  r.append(1, 0xC0 | (c >> 18));
  r.append(1, 0x80 | ((c >> 12) & 0x3F));
  r.append(1, 0x80 | ((c >> 6) & 0x3F));
  r.append(1, 0x80 | (c & 0x3F));
  return r;
}

static std::string read_string(char separator, std::istream &fromstream){
  std::string out;
  bool escaped = false;
  uint32_t fullChar = 0;
  while (fromstream.good()){
    char c;
    fromstream.get(c);
    if (c == '\\'){
      escaped = true;
      continue;
    }
    if (escaped){
      if (fullChar && c != 'u'){
        out += UTF8(fullChar >> 16);
        fullChar = 0;
      }
      switch (c){
      case 'b': out += '\b'; break;
      case 'f': out += '\f'; break;
      case 'n': out += '\n'; break;
      case 'r': out += '\r'; break;
      case 't': out += '\t'; break;
      case 'x':
        char d1, d2;
        fromstream.get(d1);
        fromstream.get(d2);
        out.append(1, (c2hex(d2) + (c2hex(d1) << 4)));
        break;
      case 'u':{
        char d1, d2, d3, d4;
        fromstream.get(d1);
        fromstream.get(d2);
        fromstream.get(d3);
        fromstream.get(d4);
        uint32_t tmpChar = (c2hex(d4) + (c2hex(d3) << 4) + (c2hex(d2) << 8) + (c2hex(d1) << 16));
        if (fullChar && (tmpChar < 0xDC00 || tmpChar > 0xDFFF)){
          // not a low surrogate - handle high surrogate separately!
          out += UTF8(fullChar >> 16);
          fullChar = 0;<commit_msg>Fix JSON parsing bug when a backslash was at the very end of a string<commit_after>/// \file json.cpp Holds all JSON-related code.

#include "json.h"
#include "bitfields.h"
#include "defines.h"
#include <arpa/inet.h> //for htonl
#include <fstream>
#include <sstream>
#include <stdint.h> //for uint64_t
#include <stdlib.h>
#include <string.h> //for memcpy

/// Construct from a root Value to iterate over.
JSON::Iter::Iter(Value &root){
  myType = root.myType;
  i = 0;
  r = &root;
  if (!root.size()){myType = JSON::EMPTY;}
  if (myType == JSON::ARRAY){aIt = root.arrVal.begin();}
  if (myType == JSON::OBJECT){oIt = root.objVal.begin();}
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
JSON::Value &JSON::Iter::operator*() const{
  if (myType == JSON::ARRAY && aIt != r->arrVal.end()){return **aIt;}
  if (myType == JSON::OBJECT && oIt != r->objVal.end()){return *(oIt->second);}
  static JSON::Value error;
  WARN_MSG("Dereferenced invalid JSON iterator");
  return error;
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
JSON::Value *JSON::Iter::operator->() const{
  return &(operator*());
}

/// True if not done iterating.
JSON::Iter::operator bool() const{
  return ((myType == JSON::ARRAY && aIt != r->arrVal.end()) ||
          (myType == JSON::OBJECT && oIt != r->objVal.end()));
}

/// Go to next iteration.
JSON::Iter &JSON::Iter::operator++(){
  if (*this){
    ++i;
    if (myType == JSON::ARRAY){++aIt;}
    if (myType == JSON::OBJECT){++oIt;}
  }
  return *this;
}

/// Return the name of the current indice.
const std::string &JSON::Iter::key() const{
  if (myType == JSON::OBJECT && *this){return oIt->first;}
  static const std::string empty;
  WARN_MSG("Got key from invalid JSON iterator");
  return empty;
}

/// Return the number of the current indice.
uint32_t JSON::Iter::num() const{
  return i;
}

/// Delete the current indice from the parent JSON::Value.
void JSON::Iter::remove(){
  if (*this){
    if (myType == JSON::ARRAY){
      r->removeMember(aIt);
      return;
    }
    if (myType == JSON::OBJECT){
      r->removeMember(oIt);
      return;
    }
  }
}

/// Construct from a root Value to iterate over.
JSON::ConstIter::ConstIter(const Value &root){
  myType = root.myType;
  i = 0;
  r = &root;
  if (!root.size()){myType = JSON::EMPTY;}
  if (myType == JSON::ARRAY){aIt = root.arrVal.begin();}
  if (myType == JSON::OBJECT){oIt = root.objVal.begin();}
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
const JSON::Value &JSON::ConstIter::operator*() const{
  if (myType == JSON::ARRAY && aIt != r->arrVal.end()){return **aIt;}
  if (myType == JSON::OBJECT && oIt != r->objVal.end()){return *(oIt->second);}
  static JSON::Value error;
  WARN_MSG("Dereferenced invalid JSON iterator");
  return error;
}

/// Dereferences into a Value reference.
/// If invalid iterator, returns an empty reference and prints a warning message.
const JSON::Value *JSON::ConstIter::operator->() const{
  return &(operator*());
}

/// True if not done iterating.
JSON::ConstIter::operator bool() const{
  return ((myType == JSON::ARRAY && aIt != r->arrVal.end()) ||
          (myType == JSON::OBJECT && oIt != r->objVal.end()));
}

/// Go to next iteration.
JSON::ConstIter &JSON::ConstIter::operator++(){
  if (*this){
    ++i;
    if (myType == JSON::ARRAY){++aIt;}
    if (myType == JSON::OBJECT){++oIt;}
  }
  return *this;
}

/// Return the name of the current indice.
const std::string &JSON::ConstIter::key() const{
  if (myType == JSON::OBJECT && *this){return oIt->first;}
  static const std::string empty;
  WARN_MSG("Got key from invalid JSON iterator");
  return empty;
}

/// Return the number of the current indice.
uint32_t JSON::ConstIter::num() const{
  return i;
}

static inline char c2hex(char c){
  if (c >= '0' && c <= '9') return c - '0';
  if (c >= 'a' && c <= 'f') return c - 'a' + 10;
  if (c >= 'A' && c <= 'F') return c - 'A' + 10;
  return 0;
}

static inline char hex2c(char c){
  if (c < 10){return '0' + c;}
  if (c < 16){return 'A' + (c - 10);}
  return '0';
}

static std::string UTF8(uint32_t c){
  std::string r;
  if (c <= 0x7F){
    r.append(1, c);
    return r;
  }
  if (c <= 0x7FF){
    r.append(1, 0xC0 | (c >> 6));
    r.append(1, 0x80 | (c & 0x3F));
    return r;
  }
  if (c <= 0x7FF){
    r.append(1, 0xC0 | (c >> 12));
    r.append(1, 0x80 | ((c >> 6) & 0x3F));
    r.append(1, 0x80 | (c & 0x3F));
    return r;
  }
  r.append(1, 0xC0 | (c >> 18));
  r.append(1, 0x80 | ((c >> 12) & 0x3F));
  r.append(1, 0x80 | ((c >> 6) & 0x3F));
  r.append(1, 0x80 | (c & 0x3F));
  return r;
}

static std::string read_string(char separator, std::istream &fromstream){
  std::string out;
  bool escaped = false;
  uint32_t fullChar = 0;
  while (fromstream.good()){
    char c;
    fromstream.get(c);
    if (!escaped && c == '\\'){
      escaped = true;
      continue;
    }
    if (escaped){
      if (fullChar && c != 'u'){
        out += UTF8(fullChar >> 16);
        fullChar = 0;
      }
      switch (c){
      case 'b': out += '\b'; break;
      case '\\': out += '\\'; break;
      case 'f': out += '\f'; break;
      case 'n': out += '\n'; break;
      case 'r': out += '\r'; break;
      case 't': out += '\t'; break;
      case 'x':
        char d1, d2;
        fromstream.get(d1);
        fromstream.get(d2);
        out.append(1, (c2hex(d2) + (c2hex(d1) << 4)));
        break;
      case 'u':{
        char d1, d2, d3, d4;
        fromstream.get(d1);
        fromstream.get(d2);
        fromstream.get(d3);
        fromstream.get(d4);
        uint32_t tmpChar = (c2hex(d4) + (c2hex(d3) << 4) + (c2hex(d2) << 8) + (c2hex(d1) << 16));
        if (fullChar && (tmpChar < 0xDC00 || tmpChar > 0xDFFF)){
          // not a low surrogate - handle high surrogate separately!
          out += UTF8(fullChar >> 16);
          fullChar = 0;
{'fraction_non_alphanumeric': 0.1941958685288952, 'fraction_numerical': 0.022915447018482898, 'mean_word_length': 3.049441246190315, 'pattern_counts': {'":': 0, '<': 49, '<?xml version=': 0, '>': 87, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '857089', 'n_tokens_mistral': 4470, 'n_tokens_neox': 4302, 'n_words': 1241}