<commit_before>//===- StreamArray.h - Array backed by an arbitrary stream ----------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H
#define LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H

#include "llvm/DebugInfo/CodeView/StreamRef.h"
#include "llvm/Support/Error.h"

#include <functional>
#include <type_traits>

namespace llvm {
namespace codeview {

/// VarStreamArrayExtractor is intended to be specialized to provide customized
/// extraction logic.  It should return the total number of bytes of the next
/// record (so that the array knows how much data to skip to get to the next
/// record, and it should initialize the second parameter with the desired
/// value type.
template <typename T> struct VarStreamArrayExtractor {
  // Method intentionally deleted.  You must provide an explicit specialization
  // with the following method implemented.  On output return `Len` should
  // contain the number of bytes to consume from the stream, and `Item` should
  // be initialized with the proper value.
  Error operator()(const StreamInterface &Stream, uint32_t &Len,
                   T &Item) const = delete;
};

/// VarStreamArray represents an array of variable length records backed by a
/// stream.  This could be a contiguous sequence of bytes in memory, it could
/// be a file on disk, or it could be a PDB stream where bytes are stored as
/// discontiguous blocks in a file.  Usually it is desirable to treat arrays
/// as contiguous blocks of memory, but doing so with large PDB files, for
/// example, could mean allocating huge amounts of memory just to allow
/// re-ordering of stream data to be contiguous before iterating over it.  By
/// abstracting this out, we need not duplicate this memory, and we can
/// iterate over arrays in arbitrarily formatted streams.
template <typename ValueType, typename Extractor> class VarStreamArrayIterator;

template <typename ValueType,
          typename Extractor = VarStreamArrayExtractor<ValueType>>
class VarStreamArray {
  friend class VarStreamArrayIterator<ValueType, Extractor>;

public:
  typedef VarStreamArrayIterator<ValueType, Extractor> Iterator;

  VarStreamArray() {}

  VarStreamArray(StreamRef Stream) : Stream(Stream) {}

  Iterator begin(bool *HadError = nullptr) const {
    return Iterator(*this, HadError);
  }

  Iterator end() const { return Iterator(); }

private:
  StreamRef Stream;
};

template <typename ValueType, typename Extractor> class VarStreamArrayIterator {
  typedef VarStreamArrayIterator<ValueType, Extractor> IterType;
  typedef VarStreamArray<ValueType, Extractor> ArrayType;

public:
  VarStreamArrayIterator(const ArrayType &Array, bool *HadError = nullptr)
      : Array(&Array), IterRef(Array.Stream), HasError(false),
        HadError(HadError) {
    auto EC = Extract(IterRef, ThisLen, ThisValue);
    if (EC) {
      consumeError(std::move(EC));
      HasError = true;
      if (HadError)
        *HadError = true;
    }
  }
  VarStreamArrayIterator() : Array(nullptr), IterRef(), HasError(false) {}
  ~VarStreamArrayIterator() {}

  bool operator==(const IterType &R) const {
    if (Array && R.Array) {
      // Both have a valid array, make sure they're same.
      assert(Array == R.Array);
      return IterRef == R.IterRef;
    }

    // Both iterators are at the end.
    if (!Array && !R.Array)
      return true;

    // One is not at the end and one is.
    return false;
  }

  bool operator!=(const IterType &R) { return !(*this == R); }<commit_msg>[CVSymbolVisitor] It's possible for an error to occur in begin()<commit_after>//===- StreamArray.h - Array backed by an arbitrary stream ----------------===//
//
//                     The LLVM Compiler Infrastructure
//
// This file is distributed under the University of Illinois Open Source
// License. See LICENSE.TXT for details.
//
//===----------------------------------------------------------------------===//

#ifndef LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H
#define LLVM_DEBUGINFO_CODEVIEW_STREAMARRAY_H

#include "llvm/DebugInfo/CodeView/StreamRef.h"
#include "llvm/Support/Error.h"

#include <functional>
#include <type_traits>

namespace llvm {
namespace codeview {

/// VarStreamArrayExtractor is intended to be specialized to provide customized
/// extraction logic.  It should return the total number of bytes of the next
/// record (so that the array knows how much data to skip to get to the next
/// record, and it should initialize the second parameter with the desired
/// value type.
template <typename T> struct VarStreamArrayExtractor {
  // Method intentionally deleted.  You must provide an explicit specialization
  // with the following method implemented.  On output return `Len` should
  // contain the number of bytes to consume from the stream, and `Item` should
  // be initialized with the proper value.
  Error operator()(const StreamInterface &Stream, uint32_t &Len,
                   T &Item) const = delete;
};

/// VarStreamArray represents an array of variable length records backed by a
/// stream.  This could be a contiguous sequence of bytes in memory, it could
/// be a file on disk, or it could be a PDB stream where bytes are stored as
/// discontiguous blocks in a file.  Usually it is desirable to treat arrays
/// as contiguous blocks of memory, but doing so with large PDB files, for
/// example, could mean allocating huge amounts of memory just to allow
/// re-ordering of stream data to be contiguous before iterating over it.  By
/// abstracting this out, we need not duplicate this memory, and we can
/// iterate over arrays in arbitrarily formatted streams.
template <typename ValueType, typename Extractor> class VarStreamArrayIterator;

template <typename ValueType,
          typename Extractor = VarStreamArrayExtractor<ValueType>>
class VarStreamArray {
  friend class VarStreamArrayIterator<ValueType, Extractor>;

public:
  typedef VarStreamArrayIterator<ValueType, Extractor> Iterator;

  VarStreamArray() {}

  VarStreamArray(StreamRef Stream) : Stream(Stream) {}

  Iterator begin(bool *HadError = nullptr) const {
    return Iterator(*this, HadError);
  }

  Iterator end() const { return Iterator(); }

private:
  StreamRef Stream;
};

template <typename ValueType, typename Extractor> class VarStreamArrayIterator {
  typedef VarStreamArrayIterator<ValueType, Extractor> IterType;
  typedef VarStreamArray<ValueType, Extractor> ArrayType;

public:
  VarStreamArrayIterator(const ArrayType &Array, bool *HadError = nullptr)
      : Array(&Array), IterRef(Array.Stream), HasError(false),
        HadError(HadError) {
    auto EC = Extract(IterRef, ThisLen, ThisValue);
    if (EC) {
      consumeError(std::move(EC));
      this->Array = nullptr;
      HasError = true;
      if (HadError)
        *HadError = true;
    }
  }
  VarStreamArrayIterator() : Array(nullptr), IterRef(), HasError(false) {}
  ~VarStreamArrayIterator() {}

  bool operator==(const IterType &R) const {
    if (Array && R.Array) {
      // Both have a valid array, make sure they're same.
      assert(Array == R.Array);
      return IterRef == R.IterRef;
    }

    // Both iterators are at the end.
    if (!Array && !R.Array)
      return true;

    // One is not at the end and one is.
    return false;
  }

  bool operator!=(const IterType &R) { return !(*this == R); }
{'fraction_non_alphanumeric': 0.11450886947312681, 'fraction_numerical': 0.0005295207836907599, 'mean_word_length': 3.24438202247191, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 26, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6806424', 'n_tokens_mistral': 2069, 'n_tokens_neox': 1976, 'n_words': 803}