<commit_before># -*- coding: utf-8 -*-
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Operations for working with string Tensors."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_parsing_ops
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import math_ops

# go/tf-wildcard-import
# pylint: disable=wildcard-import
# pylint: disable=g-bad-import-order
from tensorflow.python.ops.gen_string_ops import *
from tensorflow.python.util import compat as util_compat
from tensorflow.python.util import deprecation
from tensorflow.python.util import dispatch
from tensorflow.python.util.tf_export import tf_export
# pylint: enable=g-bad-import-order
# pylint: enable=wildcard-import


# pylint: disable=redefined-builtin
@tf_export("strings.regex_full_match")
@dispatch.add_dispatch_support
def regex_full_match(input, pattern, name=None):
  r"""Match elements of `input` with regex `pattern`.

  Args:
    input: string `Tensor`, the source strings to process.
    pattern: string or scalar string `Tensor`, regular expression to use,
      see more details at https://github.com/google/re2/wiki/Syntax
    name: Name of the op.

  Returns:
    bool `Tensor` of the same shape as `input` with match results.
  """
  if isinstance(pattern, util_compat.bytes_or_text_types):
    # When `pattern` is static through the life of the op we can
    # use a version which performs the expensive regex compilation once at
    # creation time.
    return gen_string_ops.static_regex_full_match(
        input=input, pattern=pattern, name=name)
  return gen_string_ops.regex_full_match(
      input=input, pattern=pattern, name=name)

regex_full_match.__doc__ = gen_string_ops.regex_full_match.__doc__


@tf_export(
    "strings.regex_replace", v1=["strings.regex_replace", "regex_replace"])
@deprecation.deprecated_endpoints("regex_replace")
@dispatch.add_dispatch_support
def regex_replace(input, pattern, rewrite, replace_global=True, name=None):
  r"""Replace elements of `input` matching regex `pattern` with `rewrite`.

  >>> tf.strings.regex_replace("Text with tags.<br /><b>contains html</b>",
  ...                          "<[^>]+>", " ")
  <tf.Tensor: shape=(), dtype=string, numpy=b'Text with tags.  contains html '>

  Args:
    input: string `Tensor`, the source strings to process.
    pattern: string or scalar string `Tensor`, regular expression to use,
      see more details at https://github.com/google/re2/wiki/Syntax
    rewrite: string or scalar string `Tensor`, value to use in match
      replacement, supports backslash-escaped digits (\1 to \9) can be to insert
      text matching corresponding parenthesized group.
    replace_global: `bool`, if `True` replace all non-overlapping matches,
      else replace only the first match.
    name: A name for the operation (optional).

  Returns:
    string `Tensor` of the same shape as `input` with specified replacements.
  """
  if (isinstance(pattern, util_compat.bytes_or_text_types) and
      isinstance(rewrite, util_compat.bytes_or_text_types)):
    # When `pattern` and `rewrite` are static through the life of the op we can
    # use a version which performs the expensive regex compilation once at
    # creation time.
    return gen_string_ops.static_regex_replace(
        input=input, pattern=pattern,
        rewrite=rewrite, replace_global=replace_global,
        name=name)
  return gen_string_ops.regex_replace(
      input=input, pattern=pattern,
      rewrite=rewrite, replace_global=replace_global,
      name=name)


@tf_export("strings.format")
def string_format(template, inputs, placeholder="{}", summarize=3, name=None):
  r"""Formats a string template using a list of tensors.

  Formats a string template using a list of tensors, abbreviating tensors by
  only printing the first and last `summarize` elements of each dimension
  (recursively). If formatting only one tensor into a template, the tensor does
  not have to be wrapped in a list.

  Example:
    Formatting a single-tensor template:
    ```python
    >>> tensor = tf.range(10)
    >>> formatted = tf.strings.format("tensor: {}, suffix", tensor)
    >>> expected = "tensor: [0 1 2 ... 7 8 9], suffix"
    >>> assert(formatted == expected)
    ```

    Formatting a multi-tensor template:
    ```python
    >>> tensor_one = tf.reshape(tf.range(100), [10, 10])
    >>> tensor_two = tf.range(10)
    >>> formatted = tf.strings.format("first: {}, second: {}, suffix",
    ...                               (tensor_one, tensor_two))
    >>> expected = ("first: [[0 1 2 ... 7 8 9]\n"
    ...             " [10 11 12 ... 17 18 19]\n"
    ...             " [20 21 22 ... 27 28 29]\n"
    ...             " ...\n"
    ...             " [70 71 72 ... 77 78 79]\n"
    ...             " [80 81 82 ... 87 88 89]\n"
    ...             " [90 91 92 ... 97 98 99]], second: [0 1 2 ... 7 8 9], suffix")
    >>> assert(formatted == expected)
    ```

  Args:
    template: A string template to format tensor values into.
    inputs: A list of `Tensor` objects, or a single Tensor.
      The list of tensors to format into the template string. If a solitary
      tensor is passed in, the input tensor will automatically be wrapped as a
      list.
    placeholder: An optional `string`. Defaults to `{}`.
      At each placeholder occurring in the template, a subsequent tensor
      will be inserted.
    summarize: An optional `int`. Defaults to `3`.
      When formatting the tensors, show the first and last `summarize`
      entries of each tensor dimension (recursively). If set to -1, all
      elements of the tensor will be shown.
    name: A name for the operation (optional).

  Returns:
    A scalar `Tensor` of type `string`.

  Raises:
    ValueError: if the number of placeholders does not match the number of<commit_msg>Remove explicit assert<commit_after># -*- coding: utf-8 -*-
# Copyright 2015 The TensorFlow Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# ==============================================================================

"""Operations for working with string Tensors."""

from __future__ import absolute_import
from __future__ import division
from __future__ import print_function

import numpy as np

from tensorflow.python.framework import constant_op
from tensorflow.python.framework import dtypes
from tensorflow.python.framework import ops
from tensorflow.python.framework import sparse_tensor
from tensorflow.python.framework import tensor_util
from tensorflow.python.ops import array_ops
from tensorflow.python.ops import gen_parsing_ops
from tensorflow.python.ops import gen_string_ops
from tensorflow.python.ops import math_ops

# go/tf-wildcard-import
# pylint: disable=wildcard-import
# pylint: disable=g-bad-import-order
from tensorflow.python.ops.gen_string_ops import *
from tensorflow.python.util import compat as util_compat
from tensorflow.python.util import deprecation
from tensorflow.python.util import dispatch
from tensorflow.python.util.tf_export import tf_export
# pylint: enable=g-bad-import-order
# pylint: enable=wildcard-import


# pylint: disable=redefined-builtin
@tf_export("strings.regex_full_match")
@dispatch.add_dispatch_support
def regex_full_match(input, pattern, name=None):
  r"""Match elements of `input` with regex `pattern`.

  Args:
    input: string `Tensor`, the source strings to process.
    pattern: string or scalar string `Tensor`, regular expression to use,
      see more details at https://github.com/google/re2/wiki/Syntax
    name: Name of the op.

  Returns:
    bool `Tensor` of the same shape as `input` with match results.
  """
  if isinstance(pattern, util_compat.bytes_or_text_types):
    # When `pattern` is static through the life of the op we can
    # use a version which performs the expensive regex compilation once at
    # creation time.
    return gen_string_ops.static_regex_full_match(
        input=input, pattern=pattern, name=name)
  return gen_string_ops.regex_full_match(
      input=input, pattern=pattern, name=name)

regex_full_match.__doc__ = gen_string_ops.regex_full_match.__doc__


@tf_export(
    "strings.regex_replace", v1=["strings.regex_replace", "regex_replace"])
@deprecation.deprecated_endpoints("regex_replace")
@dispatch.add_dispatch_support
def regex_replace(input, pattern, rewrite, replace_global=True, name=None):
  r"""Replace elements of `input` matching regex `pattern` with `rewrite`.

  >>> tf.strings.regex_replace("Text with tags.<br /><b>contains html</b>",
  ...                          "<[^>]+>", " ")
  <tf.Tensor: shape=(), dtype=string, numpy=b'Text with tags.  contains html '>

  Args:
    input: string `Tensor`, the source strings to process.
    pattern: string or scalar string `Tensor`, regular expression to use,
      see more details at https://github.com/google/re2/wiki/Syntax
    rewrite: string or scalar string `Tensor`, value to use in match
      replacement, supports backslash-escaped digits (\1 to \9) can be to insert
      text matching corresponding parenthesized group.
    replace_global: `bool`, if `True` replace all non-overlapping matches,
      else replace only the first match.
    name: A name for the operation (optional).

  Returns:
    string `Tensor` of the same shape as `input` with specified replacements.
  """
  if (isinstance(pattern, util_compat.bytes_or_text_types) and
      isinstance(rewrite, util_compat.bytes_or_text_types)):
    # When `pattern` and `rewrite` are static through the life of the op we can
    # use a version which performs the expensive regex compilation once at
    # creation time.
    return gen_string_ops.static_regex_replace(
        input=input, pattern=pattern,
        rewrite=rewrite, replace_global=replace_global,
        name=name)
  return gen_string_ops.regex_replace(
      input=input, pattern=pattern,
      rewrite=rewrite, replace_global=replace_global,
      name=name)


@tf_export("strings.format")
def string_format(template, inputs, placeholder="{}", summarize=3, name=None):
  r"""Formats a string template using a list of tensors.

  Formats a string template using a list of tensors, abbreviating tensors by
  only printing the first and last `summarize` elements of each dimension
  (recursively). If formatting only one tensor into a template, the tensor does
  not have to be wrapped in a list.

  Example:
    Formatting a single-tensor template:
    ```python
    >>> tensor = tf.range(10)
    >>> formatted = tf.strings.format("tensor: {}, suffix", tensor)
    >>> print(formatted)
    tf.Tensor(b'tensor: [0 1 2 ... 7 8 9], suffix', shape=(), dtype=string)
    ```

    Formatting a multi-tensor template:
    ```python
    >>> tensor_one = tf.reshape(tf.range(100), [10, 10])
    >>> tensor_two = tf.range(10)
    >>> formatted = tf.strings.format("first: {}, second: {}, suffix",
    ...                               (tensor_one, tensor_two))
    >>> print(formatted)
    tf.Tensor(b'first: [[0 1 2 ... 7 8 9]\n [10 11 12 ... 17 18 19]\n [20 21 22 ... 27 28 29]\n ...\n [70 71 72 ... 77 78 79]\n [80 81 82 ... 87 88 89]\n [90 91 92 ... 97 98 99]], second: [0 1 2 ... 7 8 9], suffix', shape=(), dtype=string)
    ```

  Args:
    template: A string template to format tensor values into.
    inputs: A list of `Tensor` objects, or a single Tensor.
      The list of tensors to format into the template string. If a solitary
      tensor is passed in, the input tensor will automatically be wrapped as a
      list.
    placeholder: An optional `string`. Defaults to `{}`.
      At each placeholder occurring in the template, a subsequent tensor
      will be inserted.
    summarize: An optional `int`. Defaults to `3`.
      When formatting the tensors, show the first and last `summarize`
      entries of each tensor dimension (recursively). If set to -1, all
      elements of the tensor will be shown.
    name: A name for the operation (optional).

  Returns:
    A scalar `Tensor` of type `string`.

  Raises:
    ValueError: if the number of placeholders does not match the number of
{'fraction_non_alphanumeric': 0.1079967630397999, 'fraction_numerical': 0.015596262782314426, 'mean_word_length': 3.6285325161729656, 'pattern_counts': {'":': 0, '<': 13, '<?xml version=': 0, '>': 69, 'https://': 4, 'lorem ipsum': 0, 'www.': 2, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 2, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '3850965', 'n_tokens_mistral': 3977, 'n_tokens_neox': 3728, 'n_words': 1429}