<commit_before>////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
////////////////////////////////////////////////////////////////////////////////

#include "AllRowsFetcher.h"

#include "Aql/AqlItemBlock.h"
#include "Aql/AqlItemMatrix.h"
#include "Aql/DependencyProxy.h"
#include "Aql/InputAqlItemRow.h"
#include "Aql/ShadowAqlItemRow.h"

using namespace arangodb;
using namespace arangodb::aql;

std::pair<ExecutionState, AqlItemMatrix const*> AllRowsFetcher::fetchAllRows() {
  switch (_dataFetchedState) {
    case ALL_DATA_FETCHED:
      // Avoid unnecessary upstream calls
      return {ExecutionState::DONE, nullptr};
    case NONE:
    case SHADOW_ROW_FETCHED: {
      auto state = fetchData();
      if (state == ExecutionState::WAITING) {
        return {state, nullptr};
      }
      _dataFetchedState = ALL_DATA_FETCHED;
      return {state, _aqlItemMatrix.get()};
    }
    case DATA_FETCH_ONGOING:
      // Invalid state, we switch between singleRow
      // and allRows fetches.
      TRI_ASSERT(false);
      // In production hand out the Matrix
      // This way the function behaves as if no single row fetch would have taken place
      _dataFetchedState = ALL_DATA_FETCHED;
      return {ExecutionState::DONE, _aqlItemMatrix.get()};
  }
  // Unreachable code
  TRI_ASSERT(false);
  return {ExecutionState::DONE, nullptr};
}

std::pair<ExecutionState, InputAqlItemRow> AllRowsFetcher::fetchRow(size_t atMost) {
  switch (_dataFetchedState) {
    case ALL_DATA_FETCHED:
      // We have already returned all data to the next shadow row
      // Avoid unnecessary upstream calls
      return {ExecutionState::DONE, InputAqlItemRow{CreateInvalidInputRowHint{}}};
    case NONE:
    case SHADOW_ROW_FETCHED: {
      // We need to get new data!
      auto state = fetchData();
      if (state == ExecutionState::WAITING) {
        return {state, InputAqlItemRow{CreateInvalidInputRowHint{}}};
      }
      TRI_ASSERT(_aqlItemMatrix != nullptr);
      _rowIndexes = _aqlItemMatrix->produceRowIndexes();
      if (_rowIndexes.empty()) {
        // We do not ahve indexes in the next block
        // So we return invalid row and can set ALL_DATA_FETCHED
        _dataFetchedState = ALL_DATA_FETCHED;
        return {ExecutionState::DONE, InputAqlItemRow{CreateInvalidInputRowHint{}}};
      }
      _nextReturn = 0;
      _dataFetchedState = DATA_FETCH_ONGOING;
      // intentionally falls through
    }
    case DATA_FETCH_ONGOING: {<commit_msg>Move warning-suppressing comment to the right place (#10191)<commit_after>////////////////////////////////////////////////////////////////////////////////
/// DISCLAIMER
///
/// Copyright 2018 ArangoDB GmbH, Cologne, Germany
///
/// Licensed under the Apache License, Version 2.0 (the "License");
/// you may not use this file except in compliance with the License.
/// You may obtain a copy of the License at
///
///     http://www.apache.org/licenses/LICENSE-2.0
///
/// Unless required by applicable law or agreed to in writing, software
/// distributed under the License is distributed on an "AS IS" BASIS,
/// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
/// See the License for the specific language governing permissions and
/// limitations under the License.
///
/// Copyright holder is ArangoDB GmbH, Cologne, Germany
///
/// @author <NAME>
////////////////////////////////////////////////////////////////////////////////

#include "AllRowsFetcher.h"

#include "Aql/AqlItemBlock.h"
#include "Aql/AqlItemMatrix.h"
#include "Aql/DependencyProxy.h"
#include "Aql/InputAqlItemRow.h"
#include "Aql/ShadowAqlItemRow.h"

using namespace arangodb;
using namespace arangodb::aql;

std::pair<ExecutionState, AqlItemMatrix const*> AllRowsFetcher::fetchAllRows() {
  switch (_dataFetchedState) {
    case ALL_DATA_FETCHED:
      // Avoid unnecessary upstream calls
      return {ExecutionState::DONE, nullptr};
    case NONE:
    case SHADOW_ROW_FETCHED: {
      auto state = fetchData();
      if (state == ExecutionState::WAITING) {
        return {state, nullptr};
      }
      _dataFetchedState = ALL_DATA_FETCHED;
      return {state, _aqlItemMatrix.get()};
    }
    case DATA_FETCH_ONGOING:
      // Invalid state, we switch between singleRow
      // and allRows fetches.
      TRI_ASSERT(false);
      // In production hand out the Matrix
      // This way the function behaves as if no single row fetch would have taken place
      _dataFetchedState = ALL_DATA_FETCHED;
      return {ExecutionState::DONE, _aqlItemMatrix.get()};
  }
  // Unreachable code
  TRI_ASSERT(false);
  return {ExecutionState::DONE, nullptr};
}

std::pair<ExecutionState, InputAqlItemRow> AllRowsFetcher::fetchRow(size_t atMost) {
  switch (_dataFetchedState) {
    case ALL_DATA_FETCHED:
      // We have already returned all data to the next shadow row
      // Avoid unnecessary upstream calls
      return {ExecutionState::DONE, InputAqlItemRow{CreateInvalidInputRowHint{}}};
    case NONE:
    case SHADOW_ROW_FETCHED: {
      // We need to get new data!
      auto state = fetchData();
      if (state == ExecutionState::WAITING) {
        return {state, InputAqlItemRow{CreateInvalidInputRowHint{}}};
      }
      TRI_ASSERT(_aqlItemMatrix != nullptr);
      _rowIndexes = _aqlItemMatrix->produceRowIndexes();
      if (_rowIndexes.empty()) {
        // We do not ahve indexes in the next block
        // So we return invalid row and can set ALL_DATA_FETCHED
        _dataFetchedState = ALL_DATA_FETCHED;
        return {ExecutionState::DONE, InputAqlItemRow{CreateInvalidInputRowHint{}}};
      }
      _nextReturn = 0;
      _dataFetchedState = DATA_FETCH_ONGOING;
    }
    // intentionally falls through
    case DATA_FETCH_ONGOING: {
{'fraction_non_alphanumeric': 0.14619069695564826, 'fraction_numerical': 0.0035543192705918716, 'mean_word_length': 2.6710153148043108, 'pattern_counts': {'":': 0, '<': 9, '<?xml version=': 0, '>': 11, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6514662', 'n_tokens_mistral': 1921, 'n_tokens_neox': 1854, 'n_words': 553}