<commit_before>/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/util/strided_slice_op.h"

#include <algorithm>
#include <array>
#include <iterator>

#include "tensorflow/core/framework/bounds_check.h"
#include "tensorflow/core/lib/core/status.h"

namespace tensorflow {
namespace {

/// Constants
constexpr int32_t kShrinkAxis = -1, kNewAxis = -2;

// Sparse slicing specification
// if one does foo[3:5, ..., -3], this will have 3 length tensors
struct StridedSliceSparseSpec {
  int64_t dims;
  int32 num_add_axis_after_ellipsis;
  const Tensor* begin_tensor;
  const Tensor* end_tensor;
  const Tensor& strides_tensor;
  const int32 begin_mask, end_mask;
  int32 ellipsis_mask;
  const int32 new_axis_mask, shrink_axis_mask;
};

// Dense slicing specification
// all ellipses and newaxis' are expanded out. So if
// foo[3:5, ..., -3] where foo is 10 dimensional,
// each inlinedVector will have 10 entries whereas the
// sparse had 3 length tensors.
struct StridedSliceDenseSpec {
  const int64_t dims;
  int32 begin_mask;
  int32 end_mask;
  bool begin_valid;
  bool end_valid;
  gtl::InlinedVector<int64_t, 4>& begin;
  gtl::InlinedVector<int64_t, 4>& end;
  gtl::InlinedVector<int64_t, 4>& strides;
  // This vector helps construct the final shape of the slice.
  // The final tensor is reduced in rank whenever a single index e.g. foo[3]
  // is called for. The final tensor increases in rank with tf.newaxis
  // entries. If an index in this array is positive, the size of the dimension
  // is obtained from canonical end-begin. Otherwise, if it is a kNewAxis,
  // it will be 1. A shrunk dimension is skipped.
  gtl::InlinedVector<int32, 4> final_shape_gather_indices;
  // This vector has the same size as final_shape_gather_indices, but it
  // remembers the sparse index that a dimension comes from, instead of dense
  // index. A -1 in this vector means there the index is not from the sparse
  // input.
  gtl::InlinedVector<int32, 4> final_shape_gather_indices_sparse;
  gtl::InlinedVector<int32, 4> input_shape_gather_indices_sparse;
  // The dense indexed shrink mask is which processing dimensions
  // should be shrunk. For example, if foo.shape = (10,10,10,10)
  // foo[3, ..., 5] has sparse_shrink_axis_mask of 0x5 and
  // dense_shrink_axis_mask of 0x9, yielding a final shape (10,10).
  int32 shrink_axis_mask;
};

}  // namespace

template <class T>
static Status TF_MUST_USE_RESULT BuildDenseSpec(
    const StridedSliceSparseSpec& sparse, StridedSliceDenseSpec* dense) {
  if (dense->dims < 0) {
    return errors::InvalidArgument("Unexpected negative dense.dims");
  }

  // Build expanded begin, end, strides, begin_mask, end_mask
  // to remove any ellipsis
  dense->begin.resize(dense->dims);<commit_msg>[tensorflow] Add a check that strided slice op strides argument has reasonable size<commit_after>/* Copyright 2015 The TensorFlow Authors. All Rights Reserved.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
==============================================================================*/

#include "tensorflow/core/util/strided_slice_op.h"

#include <algorithm>
#include <array>
#include <iterator>

#include "tensorflow/core/framework/bounds_check.h"
#include "tensorflow/core/lib/core/status.h"

namespace tensorflow {
namespace {

/// Constants
constexpr int32_t kShrinkAxis = -1, kNewAxis = -2;

// Sparse slicing specification
// if one does foo[3:5, ..., -3], this will have 3 length tensors
struct StridedSliceSparseSpec {
  int64_t dims;
  int32 num_add_axis_after_ellipsis;
  const Tensor* begin_tensor;
  const Tensor* end_tensor;
  const Tensor& strides_tensor;
  const int32 begin_mask, end_mask;
  int32 ellipsis_mask;
  const int32 new_axis_mask, shrink_axis_mask;
};

// Dense slicing specification
// all ellipses and newaxis' are expanded out. So if
// foo[3:5, ..., -3] where foo is 10 dimensional,
// each inlinedVector will have 10 entries whereas the
// sparse had 3 length tensors.
struct StridedSliceDenseSpec {
  const int64_t dims;
  int32 begin_mask;
  int32 end_mask;
  bool begin_valid;
  bool end_valid;
  gtl::InlinedVector<int64_t, 4>& begin;
  gtl::InlinedVector<int64_t, 4>& end;
  gtl::InlinedVector<int64_t, 4>& strides;
  // This vector helps construct the final shape of the slice.
  // The final tensor is reduced in rank whenever a single index e.g. foo[3]
  // is called for. The final tensor increases in rank with tf.newaxis
  // entries. If an index in this array is positive, the size of the dimension
  // is obtained from canonical end-begin. Otherwise, if it is a kNewAxis,
  // it will be 1. A shrunk dimension is skipped.
  gtl::InlinedVector<int32, 4> final_shape_gather_indices;
  // This vector has the same size as final_shape_gather_indices, but it
  // remembers the sparse index that a dimension comes from, instead of dense
  // index. A -1 in this vector means there the index is not from the sparse
  // input.
  gtl::InlinedVector<int32, 4> final_shape_gather_indices_sparse;
  gtl::InlinedVector<int32, 4> input_shape_gather_indices_sparse;
  // The dense indexed shrink mask is which processing dimensions
  // should be shrunk. For example, if foo.shape = (10,10,10,10)
  // foo[3, ..., 5] has sparse_shrink_axis_mask of 0x5 and
  // dense_shrink_axis_mask of 0x9, yielding a final shape (10,10).
  int32 shrink_axis_mask;
};

}  // namespace

template <class T>
static Status TF_MUST_USE_RESULT BuildDenseSpec(
    const StridedSliceSparseSpec& sparse, StridedSliceDenseSpec* dense) {
  if (dense->dims < 0) {
    return errors::InvalidArgument("Unexpected negative dense.dims: %d",
                                   dense->dims);
  }

  if (dense->dims >= 1024) {
    // We do not expect to see tensors with rank >= 1024, it must mean that
    // there is a bug somewhere.
    return errors::InvalidArgument("Unexpected large dense.dims: %d",
                                   dense->dims);
  }

  // Build expanded begin, end, strides, begin_mask, end_mask
  // to remove any ellipsis
  dense->begin.resize(dense->dims);
{'fraction_non_alphanumeric': 0.10716284275321768, 'fraction_numerical': 0.024062674874090653, 'mean_word_length': 3.981881533101045, 'pattern_counts': {'":': 0, '<': 25, '<?xml version=': 0, '>': 34, 'https://': 0, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5887643', 'n_tokens_mistral': 2291, 'n_tokens_neox': 2133, 'n_words': 807}