<commit_before>#ifndef CAFFE2_CORE_BLOB_H_
#define CAFFE2_CORE_BLOB_H_

#include <cstddef>
#include <vector>

#include "caffe2/core/common.h"
#include "caffe2/core/context.h"
#include "caffe2/core/registry.h"
#include "caffe2/core/typeid.h"
#include "caffe2/proto/caffe2.pb.h"
#include "glog/logging.h"

namespace caffe2 {

namespace internal {
// Destroy is a templated function that allows us to memorize the type of the
// pointer we are storing in a void*.
template <class T>
void Destroy(void* pointer) {
  delete static_cast<T*>(pointer);
}
}  // namespace internal

// Blob is a general container that hosts a pointer as well as checking its
// type, and takes charge of deleting it when the blob is deallocated. A blob
// could contain ANYTHING, although the most common case is to contain a Tensor.
class Blob {
 public:
  typedef void (*DestroyCall)(void *);

  Blob() : id_(internal::gUnknownType), pointer_(nullptr) {}

  ~Blob() { Reset(); }

  template <class T>
  inline bool IsType() const { return internal::IsTypeId<T>(id_); }
  inline string TypeName() const { return internal::TypeName(id_); }
  template <class T>
  const T& Get() const {
    CHECK(IsType<T>()) << "wrong type for the Blob instance. Expected "
                       << internal::TypeName<T>() << " got "
                       << internal::TypeName(id_);
    return *static_cast<const T*>(pointer_);
  }

  template <class T>
  T* GetMutable() {
    if (!IsType<T>()) {
      VLOG(1) << "Create new mutable object " << internal::TypeName<T>();
      if (pointer_) destroy_(pointer_);
      // If we are not of the right type, create a new instance.
      pointer_ = static_cast<void*>(new T());
      destroy_ = &internal::Destroy<T>;
    }
    id_ = internal::GetTypeId<T>();
    return static_cast<T*>(pointer_);
  }

  inline void Reset() {
    if (pointer_) {
      destroy_(pointer_);
      pointer_ = nullptr;
    }
  }

  // Serializes the current blob, if possible. This serialization uses
  // registration so we don't need to deal with multiple platform problems.
  inline string Serialize(const string& name) const;

 private:
  internal::TypeId id_;
  void* pointer_;
  DestroyCall destroy_;

  DISABLE_COPY_AND_ASSIGN(Blob);
};

// BlobSerializerBase is a class that serializes a blob to a string. This class
// exists purely for the purpose of registering type-specific serialization
// code.
class BlobSerializerBase {
 public:
  virtual string Serialize(const Blob& blob, const string& name) = 0;
};

// THe Blob serialization registry and serializer creator functions.
DECLARE_TYPED_REGISTRY(BlobSerializerRegistry, internal::TypeId,
                       BlobSerializerBase);
#define REGISTER_BLOB_SERIALIZER(name, id, ...) \
  REGISTER_TYPED_CLASS(BlobSerializerRegistry, name, id, __VA_ARGS__)
// Creates an operator with the given operator definition.
inline BlobSerializerBase* CreateSerializer(internal::TypeId id) {
  return BlobSerializerRegistry()->Create(id);
}

// The blob serialization member function implementation.
inline string Blob::Serialize(const string& name) const {
  std::unique_ptr<BlobSerializerBase> serializer(CreateSerializer(id_));
  return serializer->Serialize(*this, name);
}


template <typename dtype, class Context>
class Tensor {
 public:
  Tensor() : ndim_(0), size_(0), data_(nullptr) {}

  // Creates a tensor. The actual data allocation is going to be carried out
  // till the first time mutable_data() is called, so there is no overhead of
  // creating multiple tensors just as placeholders (although I haven't got a
  // clear idea where such cases would happen).
  explicit Tensor(const vector<int>& dims)
      : data_(nullptr) {
    Reshape(dims);
  }

  template <class SrcContext, class ContextForCopy>
  Tensor(const Tensor<dtype, SrcContext>& src, ContextForCopy* context)
      : data_(nullptr) {
    Reshape(src.dims());
    context->template Copy<dtype, Context, SrcContext>(
        mutable_data(), src.data(), src.size());
  }

  // Creates a tensor, and fills its contents with the given values. We need to
  // have a context passed in as the copy function is device dependent.
  Tensor(const vector<int>& dims, vector<dtype> values, Context* context)
      : data_(nullptr) {
    Reshape(dims);
    CHECK_EQ(values.size(), size_);
    context->template Copy<dtype, Context, CPUContext>(
        mutable_data(), values.data(), values.size());
  }

  // Special case of above: create a tensor of shape 1, and the given value.
  Tensor(const dtype& value, Context* context)
      : data_(nullptr) {
    Reshape(std::vector<int>(1, 1));
    context->template Copy<dtype, Context, CPUContext>(
        mutable_data(), &value, 1);
  }

  virtual ~Tensor() {}

  void Reshape(const vector<int>& dims) {
    CHECK_GT(dims.size(), 0);
    dims_ = dims;
    ndim_ = dims_.size();<commit_msg>Tensor constructor: values arg is const<commit_after>#ifndef CAFFE2_CORE_BLOB_H_
#define CAFFE2_CORE_BLOB_H_

#include <cstddef>
#include <vector>

#include "caffe2/core/common.h"
#include "caffe2/core/context.h"
#include "caffe2/core/registry.h"
#include "caffe2/core/typeid.h"
#include "caffe2/proto/caffe2.pb.h"
#include "glog/logging.h"

namespace caffe2 {

namespace internal {
// Destroy is a templated function that allows us to memorize the type of the
// pointer we are storing in a void*.
template <class T>
void Destroy(void* pointer) {
  delete static_cast<T*>(pointer);
}
}  // namespace internal

// Blob is a general container that hosts a pointer as well as checking its
// type, and takes charge of deleting it when the blob is deallocated. A blob
// could contain ANYTHING, although the most common case is to contain a Tensor.
class Blob {
 public:
  typedef void (*DestroyCall)(void *);

  Blob() : id_(internal::gUnknownType), pointer_(nullptr) {}

  ~Blob() { Reset(); }

  template <class T>
  inline bool IsType() const { return internal::IsTypeId<T>(id_); }
  inline string TypeName() const { return internal::TypeName(id_); }
  template <class T>
  const T& Get() const {
    CHECK(IsType<T>()) << "wrong type for the Blob instance. Expected "
                       << internal::TypeName<T>() << " got "
                       << internal::TypeName(id_);
    return *static_cast<const T*>(pointer_);
  }

  template <class T>
  T* GetMutable() {
    if (!IsType<T>()) {
      VLOG(1) << "Create new mutable object " << internal::TypeName<T>();
      if (pointer_) destroy_(pointer_);
      // If we are not of the right type, create a new instance.
      pointer_ = static_cast<void*>(new T());
      destroy_ = &internal::Destroy<T>;
    }
    id_ = internal::GetTypeId<T>();
    return static_cast<T*>(pointer_);
  }

  inline void Reset() {
    if (pointer_) {
      destroy_(pointer_);
      pointer_ = nullptr;
    }
  }

  // Serializes the current blob, if possible. This serialization uses
  // registration so we don't need to deal with multiple platform problems.
  inline string Serialize(const string& name) const;

 private:
  internal::TypeId id_;
  void* pointer_;
  DestroyCall destroy_;

  DISABLE_COPY_AND_ASSIGN(Blob);
};

// BlobSerializerBase is a class that serializes a blob to a string. This class
// exists purely for the purpose of registering type-specific serialization
// code.
class BlobSerializerBase {
 public:
  virtual string Serialize(const Blob& blob, const string& name) = 0;
};

// THe Blob serialization registry and serializer creator functions.
DECLARE_TYPED_REGISTRY(BlobSerializerRegistry, internal::TypeId,
                       BlobSerializerBase);
#define REGISTER_BLOB_SERIALIZER(name, id, ...) \
  REGISTER_TYPED_CLASS(BlobSerializerRegistry, name, id, __VA_ARGS__)
// Creates an operator with the given operator definition.
inline BlobSerializerBase* CreateSerializer(internal::TypeId id) {
  return BlobSerializerRegistry()->Create(id);
}

// The blob serialization member function implementation.
inline string Blob::Serialize(const string& name) const {
  std::unique_ptr<BlobSerializerBase> serializer(CreateSerializer(id_));
  return serializer->Serialize(*this, name);
}


template <typename dtype, class Context>
class Tensor {
 public:
  Tensor() : ndim_(0), size_(0), data_(nullptr) {}

  // Creates a tensor. The actual data allocation is going to be carried out
  // till the first time mutable_data() is called, so there is no overhead of
  // creating multiple tensors just as placeholders (although I haven't got a
  // clear idea where such cases would happen).
  explicit Tensor(const vector<int>& dims)
      : data_(nullptr) {
    Reshape(dims);
  }

  template <class SrcContext, class ContextForCopy>
  Tensor(const Tensor<dtype, SrcContext>& src, ContextForCopy* context)
      : data_(nullptr) {
    Reshape(src.dims());
    context->template Copy<dtype, Context, SrcContext>(
        mutable_data(), src.data(), src.size());
  }

  // Creates a tensor, and fills its contents with the given values. We need to
  // have a context passed in as the copy function is device dependent.
  Tensor(const vector<int>& dims, const vector<dtype>& values, Context* context)
      : data_(nullptr) {
    Reshape(dims);
    CHECK_EQ(values.size(), size_);
    context->template Copy<dtype, Context, CPUContext>(
        mutable_data(), values.data(), values.size());
  }

  // Special case of above: create a tensor of shape 1, and the given value.
  Tensor(const dtype& value, Context* context)
      : data_(nullptr) {
    Reshape(std::vector<int>(1, 1));
    context->template Copy<dtype, Context, CPUContext>(
        mutable_data(), &value, 1);
  }

  virtual ~Tensor() {}

  void Reshape(const vector<int>& dims) {
    CHECK_GT(dims.size(), 0);
    dims_ = dims;
    ndim_ = dims_.size();
{'fraction_non_alphanumeric': 0.11799773871929284, 'fraction_numerical': 0.0037002775208140612, 'mean_word_length': 3.776632302405498, 'pattern_counts': {'":': 0, '<': 85, '<?xml version=': 0, '>': 71, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3554439', 'n_tokens_mistral': 2946, 'n_tokens_neox': 2905, 'n_words': 951}