<commit_before>////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
////////////////////////////////////////////////////////////////////////////////

#include "RocksDBEdgeIndex.h"
#include "Aql/AstNode.h"
#include "Aql/SortCondition.h"
#include "Basics/Exceptions.h"
#include "Basics/LocalTaskQueue.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringRef.h"
#include "Indexes/SimpleAttributeEqualityMatcher.h"
#include "Transaction/Context.h"
#include "Transaction/Helpers.h"
#include "Transaction/Methods.h"
#include "VocBase/LogicalCollection.h"

#include "RocksDBEngine/RocksDBCollection.h"
#include "RocksDBEngine/RocksDBCommon.h"
#include "RocksDBEngine/RocksDBKey.h"
#include "RocksDBEngine/RocksDBKeyBounds.h"
#include "RocksDBEngine/RocksDBToken.h"
#include "RocksDBEngine/RocksDBTransactionState.h"
#include "RocksDBEngine/RocksDBTypes.h"
#include "RocksDBEngine/RocksDBCommon.h"

#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/utilities/transaction_db.h>

#include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h>

using namespace arangodb;
using namespace arangodb::basics;

/// @brief hard-coded vector of the index attributes
/// note that the attribute names must be hard-coded here to avoid an init-order
/// fiasco with StaticStrings::FromString etc.
/*static std::vector<std::vector<arangodb::basics::AttributeName>> const
    IndexAttributes{{arangodb::basics::AttributeName("_from", false)},
                    {arangodb::basics::AttributeName("_to", false)}};*/

RocksDBEdgeIndexIterator::RocksDBEdgeIndexIterator(
    LogicalCollection* collection, transaction::Methods* trx,
    ManagedDocumentResult* mmdr, arangodb::RocksDBEdgeIndex const* index,
    std::unique_ptr<VPackBuilder>& keys)
    : IndexIterator(collection, trx, mmdr, index),
      _keys(keys.get()),
      _iterator(_keys->slice()),
      _index(index) {
  keys.release();  // now we have ownership for _keys
}

RocksDBEdgeIndexIterator::~RocksDBEdgeIndexIterator() {
  if (_keys != nullptr) {
    // return the VPackBuilder to the transaction context
    _trx->transactionContextPtr()->returnBuilder(_keys.release());
  }
}

bool RocksDBEdgeIndexIterator::next(TokenCallback const& cb, size_t limit) {

  if (limit == 0 || !_iterator.valid()) {
    // No limit no data, or we are actually done. The last call should have
    // returned false
    TRI_ASSERT(limit > 0);  // Someone called with limit == 0. Api broken
    return false;
  }

  // aquire rocksdb transaction
  RocksDBTransactionState* state = rocksutils::toRocksTransactionState(_trx);
  rocksdb::Transaction* rtrx = state->rocksTransaction();
  auto rocksColl = RocksDBCollection::toRocksDBCollection(_collection);

  while (limit > 0) {
    VPackSlice fromTo = _iterator.value();
    if (fromTo.isObject()) {
      fromTo = fromTo.get(StaticStrings::IndexEq);
    }
    TRI_ASSERT(fromTo.isString());

    RocksDBKeyBounds prefix = RocksDBKeyBounds::EdgeIndexVertex(
        _index->_objectId, fromTo.copyString());

    std::unique_ptr<rocksdb::Iterator> iter(
        rtrx->GetIterator(state->readOptions()));

    rocksdb::Slice rSlice(prefix.start());
    iter->Seek(rSlice);
    while (iter->Valid() && iter->key().starts_with(rSlice)) {
      TRI_ASSERT(iter->key().size() > rSlice.size());
      size_t edgeKeySize = iter->key().size() - rSlice.size();
      const char* edgeKey = iter->key().data() + rSlice.size();

      // aquire the document token through the primary index
      RocksDBToken token;
      Result res = rocksColl->lookupDocumentToken(
          _trx, StringRef(edgeKey, edgeKeySize), token);
      if (res.ok()) {
        cb(token);
        if (--limit == 0) {
          break;
        }
      }  // TODO do we need to handle failed lookups here?
<commit_msg>Edge index compliant with design doc<commit_after>////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2014-2016 ArangoDB GmbH, Cologne, Germany
/// Copyright 2004-2014 triAGENS GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
////////////////////////////////////////////////////////////////////////////////

#include "RocksDBEdgeIndex.h"
#include "Aql/AstNode.h"
#include "Aql/SortCondition.h"
#include "Basics/Exceptions.h"
#include "Basics/LocalTaskQueue.h"
#include "Basics/StaticStrings.h"
#include "Basics/StringRef.h"
#include "Indexes/SimpleAttributeEqualityMatcher.h"
#include "Transaction/Context.h"
#include "Transaction/Helpers.h"
#include "Transaction/Methods.h"
#include "VocBase/LogicalCollection.h"

#include "RocksDBEngine/RocksDBCollection.h"
#include "RocksDBEngine/RocksDBCommon.h"
#include "RocksDBEngine/RocksDBKey.h"
#include "RocksDBEngine/RocksDBKeyBounds.h"
#include "RocksDBEngine/RocksDBToken.h"
#include "RocksDBEngine/RocksDBTransactionState.h"
#include "RocksDBEngine/RocksDBTypes.h"
#include "RocksDBEngine/RocksDBCommon.h"

#include <rocksdb/db.h>
#include <rocksdb/options.h>
#include <rocksdb/slice.h>
#include <rocksdb/utilities/transaction_db.h>

#include <velocypack/Iterator.h>
#include <velocypack/velocypack-aliases.h>

using namespace arangodb;
using namespace arangodb::basics;

/// @brief hard-coded vector of the index attributes
/// note that the attribute names must be hard-coded here to avoid an init-order
/// fiasco with StaticStrings::FromString etc.
/*static std::vector<std::vector<arangodb::basics::AttributeName>> const
    IndexAttributes{{arangodb::basics::AttributeName("_from", false)},
                    {arangodb::basics::AttributeName("_to", false)}};*/

RocksDBEdgeIndexIterator::RocksDBEdgeIndexIterator(
    LogicalCollection* collection, transaction::Methods* trx,
    ManagedDocumentResult* mmdr, arangodb::RocksDBEdgeIndex const* index,
    std::unique_ptr<VPackBuilder>& keys)
    : IndexIterator(collection, trx, mmdr, index),
      _keys(keys.get()),
      _iterator(_keys->slice()),
      _index(index) {
  keys.release();  // now we have ownership for _keys
}

RocksDBEdgeIndexIterator::~RocksDBEdgeIndexIterator() {
  if (_keys != nullptr) {
    // return the VPackBuilder to the transaction context
    _trx->transactionContextPtr()->returnBuilder(_keys.release());
  }
}

bool RocksDBEdgeIndexIterator::next(TokenCallback const& cb, size_t limit) {

  if (limit == 0 || !_iterator.valid()) {
    // No limit no data, or we are actually done. The last call should have
    // returned false
    TRI_ASSERT(limit > 0);  // Someone called with limit == 0. Api broken
    return false;
  }

  // aquire rocksdb transaction
  RocksDBTransactionState* state = rocksutils::toRocksTransactionState(_trx);
  rocksdb::Transaction* rtrx = state->rocksTransaction();
  auto rocksColl = RocksDBCollection::toRocksDBCollection(_collection);

  while (limit > 0) {
    VPackSlice fromTo = _iterator.value();
    if (fromTo.isObject()) {
      fromTo = fromTo.get(StaticStrings::IndexEq);
    }
    TRI_ASSERT(fromTo.isString());

    RocksDBKeyBounds bounds = RocksDBKeyBounds::EdgeIndexVertex(
        _index->_objectId, fromTo.copyString());

    std::unique_ptr<rocksdb::Iterator> iter(
        rtrx->GetIterator(state->readOptions()));

    rocksdb::Slice prefix = bounds.start();
    iter->Seek(prefix);
    while (iter->Valid() && iter->key().starts_with(prefix)) {
      TRI_ASSERT(iter->key().size() > prefix.size());
      StringRef edgeKey = RocksDBKey::primaryKey(iter->key());

      // aquire the document token through the primary index
      RocksDBToken token;
      Result res = rocksColl->lookupDocumentToken( _trx, edgeKey, token);
      if (res.ok()) {
        cb(token);
        if (--limit == 0) {
          break;
        }
      }  // TODO do we need to handle failed lookups here?

{'fraction_non_alphanumeric': 0.16137105549510336, 'fraction_numerical': 0.00544069640914037, 'mean_word_length': 3.4443907156673115, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 58, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3388612', 'n_tokens_mistral': 2812, 'n_tokens_neox': 2718, 'n_words': 691}