<commit_before>#pragma once

#include <cstdint>
#include <iostream>
#include <string>

// #include "exception.h"

namespace marian {

/**
 * @brief Represents the size of each dimension in a tensor.
 *
 * Note: this class currently is hard-coded to four dimensions.
 */

struct Shape {
  public:
    std::vector<int> shape_;
    std::vector<int> stride_;
    std::vector<int> bstride_;

  public:
    Shape() : shape_{1}, stride_{1}, bstride_{0} {}

    Shape(std::initializer_list<int> il) : Shape() {
      shape_.resize(il.size());
      std::copy(il.begin(), il.end(), begin());
      updateStrides();
    }

    void resize(size_t n) {
      shape_.resize(n, 1);
      updateStrides();
    }

    const int* data() const {
      return shape_.data();
    }

    void updateStrides() {
      stride_.resize(shape_.size());
      bstride_.resize(shape_.size());

      stride_.back() = 1;
      bstride_.back() = shape_.back() == 1 ? 0 : stride_.back();

      for(int i = size() - 2; i >= 0; --i) {
        stride_[i] = stride_[i + 1] * shape_[i + 1];
        bstride_[i] = shape_[i] == 1 ? 0 : stride_[i];
      }
    }

    Shape(const Shape& shape) : Shape() {
      shape_.resize(shape.size());
      std::copy(shape.begin(), shape.end(), begin());
      updateStrides();
    }

    inline void set(int i, int val) {
      dim(i) = val;
      updateStrides();
    }

    inline int& dim(int i) {
      if(i >= 0)
        return shape_[i];
      else
        return shape_[size() + i];
    }

    inline const int& dim(int i) const { return const_cast<Shape&>(*this).dim(i); }

    inline int operator[](int i) { return dim(i); }

    inline int operator[](int i) const { return dim(i); }

    inline int& back() { return shape_.back(); }

    inline int stride(int i) const {
      if(i >= 0)
        return stride_[i];
      else
        return stride_[size() + i];
    }

    inline int bstride(int i) const {
      if(i >= 0)
        return bstride_[i];
      else
        return bstride_[size() + i];
    }

    inline size_t size() const { return shape_.size(); }
<commit_msg>error message<commit_after>#pragma once

#include <cstdint>
#include <iostream>
#include <string>

// #include "exception.h"

namespace marian {

/**
 * @brief Represents the size of each dimension in a tensor.
 *
 * Note: this class currently is hard-coded to four dimensions.
 */

struct Shape {
  public:
    std::vector<int> shape_;
    std::vector<int> stride_;
    std::vector<int> bstride_;

  public:
    Shape() : shape_{1}, stride_{1}, bstride_{0} {}

    Shape(std::initializer_list<int> il) : Shape() {
      shape_.resize(il.size());
      std::copy(il.begin(), il.end(), begin());
      updateStrides();
    }

    void resize(size_t n) {
      shape_.resize(n, 1);
      updateStrides();
    }

    const int* data() const {
      return shape_.data();
    }

    void updateStrides() {
      stride_.resize(shape_.size());
      bstride_.resize(shape_.size());

      stride_.back() = 1;
      bstride_.back() = shape_.back() == 1 ? 0 : stride_.back();

      for(int i = size() - 2; i >= 0; --i) {
        stride_[i] = stride_[i + 1] * shape_[i + 1];
        bstride_[i] = shape_[i] == 1 ? 0 : stride_[i];
      }
    }

    Shape(const Shape& shape) : Shape() {
      shape_.resize(shape.size());
      std::copy(shape.begin(), shape.end(), begin());
      updateStrides();
    }

    inline void set(int i, int val) {
      dim(i) = val;
      updateStrides();
    }

    inline int& dim(int i) {
      if(i >= 0) {
        ABORT_IF(i >= size(),
                 "Index {} is out of bounds, shape has {} dimension", i, size());
        return shape_[i];
      }
      else {
        ABORT_IF((int)size() + i < 0,
                 "Negative index {} is out of bounds, shape has {} dimension", i, size());
        return shape_[size() + i];
      }
    }

    inline const int& dim(int i) const { return const_cast<Shape&>(*this).dim(i); }

    inline int operator[](int i) { return dim(i); }

    inline int operator[](int i) const { return dim(i); }

    inline int& back() { return shape_.back(); }

    inline int stride(int i) const {
      if(i >= 0)
        return stride_[i];
      else
        return stride_[size() + i];
    }

    inline int bstride(int i) const {
      if(i >= 0)
        return bstride_[i];
      else
        return bstride_[size() + i];
    }

    inline size_t size() const { return shape_.size(); }

{'fraction_non_alphanumeric': 0.1644514530299617, 'fraction_numerical': 0.007434106780806488, 'mean_word_length': 2.3208676140613314, 'pattern_counts': {'":': 0, '<': 20, '<?xml version=': 0, '>': 28, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5351832', 'n_tokens_mistral': 1518, 'n_tokens_neox': 1437, 'n_words': 389}