<commit_before>/* * keyvi - A key value store.
 *
 * Copyright 2015 <NAME><<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * entry_iterator.h
 *
 *  Created on: May 12, 2014
 *      Author: hendrik
 */

#ifndef ENTRY_ITERATOR_H_
#define ENTRY_ITERATOR_H_

#include "dictionary/fsa/automata.h"

//#define ENABLE_TRACING
#include "dictionary/util/trace.h"

namespace keyvi {
namespace dictionary {
namespace fsa {

class EntryIterator {

 public:
  EntryIterator()
      : current_state_(0),
        current_value_(0),
        fsa_(0) {
  }

  EntryIterator(automata_t f) : EntryIterator(f, f->GetStartState()) {}

  EntryIterator(automata_t f, int start_state)
      : fsa_(f) {
    current_state_ = start_state;
    state_vector_traversal_stack_.resize(10);
    transitions_vector_traversal_stack_.resize(10);

    fsa_->GetOutGoingTransitions(start_state, state_vector_traversal_stack_[0], transitions_vector_traversal_stack_[0]);

    vector_offset_traversal_stack_.push_back(0);
    traversal_stack_.push_back(transitions_vector_traversal_stack_[0][0]);
    TraverseToNextFinalState();
  }

  std::string GetKey() {
    TRACE ("depth: %d", current_depth_);
    return std::string((const char*) traversal_stack_.data(), current_depth_ + 1);
  }

  void WriteKey(std::ostream& stream){
    stream.write((const char*) traversal_stack_.data(), current_depth_ + 1);
  }

  uint64_t GetValueId() const {
    return current_value_;
  }

  internal::IValueStoreReader::attributes_t GetValueAsAttributeVector() const {
    return fsa_->GetValueAsAttributeVector(current_value_);
  }

  std::string GetValueAsString() const {
    return fsa_->GetValueAsString(current_value_);
  }

  EntryIterator &operator=(const EntryIterator &other) {
    fsa_ = other.fsa_;
    current_state_ = other.current_state_;
    current_value_ = other.current_value_;
    traversal_stack_ = other.traversal_stack_;
    state_vector_traversal_stack_ = other.state_vector_traversal_stack_;
    transitions_vector_traversal_stack_ = other.transitions_vector_traversal_stack_;
    vector_offset_traversal_stack_ = other.vector_offset_traversal_stack_;
    current_depth_ = other.current_depth_;
    return *this;
  }

  EntryIterator &operator++() {
    TraverseToNextFinalState();
    return *this;
  }

  bool operator==(const EntryIterator &other) const {
    return (current_state_ == other.current_state_);
  }

  bool operator!=(const EntryIterator &other) const {
    return !(operator==(other));
  }

 private:
  void TraverseToNextFinalState() {

    if (current_state_ == 0) {
      return;
    }
<commit_msg>EntryIterator to recognize empty FSAs. Fixes #41.<commit_after>/* * keyvi - A key value store.
 *
 * Copyright 2015 <NAME><<EMAIL>>
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

/*
 * entry_iterator.h
 *
 *  Created on: May 12, 2014
 *      Author: hendrik
 */

#ifndef ENTRY_ITERATOR_H_
#define ENTRY_ITERATOR_H_

#include "dictionary/fsa/automata.h"

//#define ENABLE_TRACING
#include "dictionary/util/trace.h"

namespace keyvi {
namespace dictionary {
namespace fsa {

class EntryIterator {

 public:
  EntryIterator()
      : current_state_(0),
        current_value_(0),
        fsa_(nullptr) {
  }

  EntryIterator(automata_t f) : EntryIterator(f, f->GetStartState()) {}

  EntryIterator(automata_t f, int start_state)
      : fsa_(f) {
    if (f && f->GetNumberOfKeys() > 0) {
      current_state_ = start_state;
      state_vector_traversal_stack_.resize(10);
      transitions_vector_traversal_stack_.resize(10);

      fsa_->GetOutGoingTransitions(start_state, state_vector_traversal_stack_[0], transitions_vector_traversal_stack_[0]);

      vector_offset_traversal_stack_.push_back(0);
      // If there are no transitions from the start state...
      if (transitions_vector_traversal_stack_[0].size() > 0) {
        traversal_stack_.push_back(transitions_vector_traversal_stack_[0][0]);
        TraverseToNextFinalState();
      } else {
        Clear();
      }
    } else {
      Clear();
    }
  }

  std::string GetKey() {
    TRACE ("depth: %d", current_depth_);
    return std::string((const char*) traversal_stack_.data(), current_depth_ + 1);
  }

  void WriteKey(std::ostream& stream){
    stream.write((const char*) traversal_stack_.data(), current_depth_ + 1);
  }

  uint64_t GetValueId() const {
    return current_value_;
  }

  internal::IValueStoreReader::attributes_t GetValueAsAttributeVector() const {
    return fsa_->GetValueAsAttributeVector(current_value_);
  }

  std::string GetValueAsString() const {
    return fsa_->GetValueAsString(current_value_);
  }

  EntryIterator &operator=(const EntryIterator &other) {
    fsa_ = other.fsa_;
    current_state_ = other.current_state_;
    current_value_ = other.current_value_;
    traversal_stack_ = other.traversal_stack_;
    state_vector_traversal_stack_ = other.state_vector_traversal_stack_;
    transitions_vector_traversal_stack_ = other.transitions_vector_traversal_stack_;
    vector_offset_traversal_stack_ = other.vector_offset_traversal_stack_;
    current_depth_ = other.current_depth_;
    return *this;
  }

  EntryIterator &operator++() {
    TraverseToNextFinalState();
    return *this;
  }

  bool operator==(const EntryIterator &other) const {
    return (current_state_ == other.current_state_);
  }

  bool operator!=(const EntryIterator &other) const {
    return !(operator==(other));
  }

 private:
  /** Cleares the iterator, i.e. makes it equal to the empty iterator. */
  void Clear() {
    fsa_ = nullptr;
    current_state_ = 0;
    current_state_ = 0;
  }

  void TraverseToNextFinalState() {

    if (current_state_ == 0) {
      return;
    }

{'fraction_non_alphanumeric': 0.10312359886414586, 'fraction_numerical': 0.010162905395307129, 'mean_word_length': 4.031578947368421, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 20, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3622156', 'n_tokens_mistral': 2152, 'n_tokens_neox': 2052, 'n_words': 487}