<commit_before>#include "simdjson/parsedjson.h"
#include "simdjson/common_defs.h"

ParsedJson::iterator::iterator(ParsedJson &pj_) : pj(pj_), depth(0), location(0), tape_length(0), depthindex(nullptr) {
        if(pj.isValid()) {
            depthindex = new scopeindex_t[pj.depthcapacity];
            if(depthindex == nullptr) { return;
}
            depthindex[0].start_of_scope = location;
            current_val = pj.tape[location++];
            current_type = (current_val >> 56);
            depthindex[0].scope_type = current_type;
            if (current_type == 'r') {
              tape_length = current_val & JSONVALUEMASK;
              if(location < tape_length) {
                current_val = pj.tape[location];
                current_type = (current_val >> 56);
                depth++;
                depthindex[depth].start_of_scope = location;
                depthindex[depth].scope_type = current_type;
              }
            }
        }
    }

ParsedJson::iterator::~iterator() {
      delete[] depthindex;
}

ParsedJson::iterator::iterator(const iterator &o):
    pj(o.pj), depth(o.depth), location(o.location),
    tape_length(o.tape_length), current_type(o.current_type),
    current_val(o.current_val), depthindex(nullptr) {
    depthindex = new scopeindex_t[pj.depthcapacity];
    if(depthindex != nullptr) {
        memcpy(o.depthindex, depthindex, pj.depthcapacity * sizeof(depthindex[0]));
    } else {
        tape_length = 0;
    }
}

ParsedJson::iterator::iterator(iterator &&o):
      pj(o.pj), depth(o.depth), location(o.location),
      tape_length(o.tape_length), current_type(o.current_type),
      current_val(o.current_val), depthindex(o.depthindex) {
        o.depthindex = nullptr;// we take ownership
}

WARN_UNUSED
bool ParsedJson::iterator::isOk() const {
      return location < tape_length;
}

// useful for debuging purposes
size_t ParsedJson::iterator::get_tape_location() const {
    return location;
}

// useful for debuging purposes
size_t ParsedJson::iterator::get_tape_length() const {
    return tape_length;
}

// returns the current depth (start at 1 with 0 reserved for the fictitious root node)
size_t ParsedJson::iterator::get_depth() const {
    return depth;
}

// A scope is a series of nodes at the same depth, typically it is either an object ({) or an array ([).
// The root node has type 'r'.
uint8_t ParsedJson::iterator::get_scope_type() const {
    return depthindex[depth].scope_type;
}

bool ParsedJson::iterator::move_forward() {
    if(location + 1 >= tape_length) {
    return false; // we are at the end!
    }
    // we are entering a new scope
    if ((current_type == '[') || (current_type == '{')){
    depth++;
    depthindex[depth].start_of_scope = location;
    depthindex[depth].scope_type = current_type;
    }
    location = location + 1;
    current_val = pj.tape[location];
    current_type = (current_val >> 56);
    // if we encounter a scope closure, we need to move up
    while ((current_type == ']') || (current_type == '}')) {
    if(location + 1 >= tape_length) {
        return false; // we are at the end!
    }
    depth--;
    if(depth == 0) {
        return false; // should not be necessary
    }
    location = location + 1;
    current_val = pj.tape[location];
    current_type = (current_val >> 56);
    }
    return true;
}

uint8_t ParsedJson::iterator::get_type()  const {
    return current_type;
}


int64_t ParsedJson::iterator::get_integer()  const {
    if(location + 1 >= tape_length) { return 0;// default value in case of error
}
    return static_cast<int64_t>(pj.tape[location + 1]);
}<commit_msg>Changes to the behaviour of move_forward to make it suitable for iteration. (Closes #73) (#103)<commit_after>#include "simdjson/parsedjson.h"
#include "simdjson/common_defs.h"

ParsedJson::iterator::iterator(ParsedJson &pj_) : pj(pj_), depth(0), location(0), tape_length(0), depthindex(nullptr) {
        if(pj.isValid()) {
            depthindex = new scopeindex_t[pj.depthcapacity];
            if(depthindex == nullptr) { return;
}
            depthindex[0].start_of_scope = location;
            current_val = pj.tape[location++];
            current_type = (current_val >> 56);
            depthindex[0].scope_type = current_type;
            if (current_type == 'r') {
              tape_length = current_val & JSONVALUEMASK;
              if(location < tape_length) {
                current_val = pj.tape[location];
                current_type = (current_val >> 56);
                depth++;
                depthindex[depth].start_of_scope = location;
                depthindex[depth].scope_type = current_type;
              }
            }
        }
    }

ParsedJson::iterator::~iterator() {
      delete[] depthindex;
}

ParsedJson::iterator::iterator(const iterator &o):
    pj(o.pj), depth(o.depth), location(o.location),
    tape_length(o.tape_length), current_type(o.current_type),
    current_val(o.current_val), depthindex(nullptr) {
    depthindex = new scopeindex_t[pj.depthcapacity];
    if(depthindex != nullptr) {
        memcpy(o.depthindex, depthindex, pj.depthcapacity * sizeof(depthindex[0]));
    } else {
        tape_length = 0;
    }
}

ParsedJson::iterator::iterator(iterator &&o):
      pj(o.pj), depth(o.depth), location(o.location),
      tape_length(o.tape_length), current_type(o.current_type),
      current_val(o.current_val), depthindex(o.depthindex) {
        o.depthindex = nullptr;// we take ownership
}

WARN_UNUSED
bool ParsedJson::iterator::isOk() const {
      return location < tape_length;
}

// useful for debuging purposes
size_t ParsedJson::iterator::get_tape_location() const {
    return location;
}

// useful for debuging purposes
size_t ParsedJson::iterator::get_tape_length() const {
    return tape_length;
}

// returns the current depth (start at 1 with 0 reserved for the fictitious root node)
size_t ParsedJson::iterator::get_depth() const {
    return depth;
}

// A scope is a series of nodes at the same depth, typically it is either an object ({) or an array ([).
// The root node has type 'r'.
uint8_t ParsedJson::iterator::get_scope_type() const {
    return depthindex[depth].scope_type;
}

bool ParsedJson::iterator::move_forward() {
    if(location + 1 >= tape_length) {
        return false; // we are at the end!
    }

    if ((current_type == '[') || (current_type == '{')){
        // We are entering a new scope
        depth++;
        depthindex[depth].start_of_scope = location;
        depthindex[depth].scope_type = current_type;
    } else if ((current_type == ']') || (current_type == '}')) {
        // Leaving a scope.
        depth--;
        if(depth == 0) {
            // Should not be necessary
            return false;
        }
    } else if ((current_type == 'd') || (current_type == 'l')) {
        // d and l types use 2 locations on the tape, not just one.
        location += 1;
    }

    location += 1;
    current_val = pj.tape[location];
    current_type = (current_val >> 56);
    return true;
}

uint8_t ParsedJson::iterator::get_type()  const {
    return current_type;
}


int64_t ParsedJson::iterator::get_integer()  const {
    if(location + 1 >= tape_length) { return 0;// default value in case of error
}
    return static_cast<int64_t>(pj.tape[location + 1]);
}
{'fraction_non_alphanumeric': 0.13076817746131728, 'fraction_numerical': 0.008900451869094892, 'mean_word_length': 2.9142550911039655, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 24, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5688340', 'n_tokens_mistral': 2342, 'n_tokens_neox': 2299, 'n_words': 527}