<commit_before>import numpy as np

from pandas.compat import zip
from pandas.core.dtypes.generic import ABCSeries, ABCIndex
from pandas.core.dtypes.missing import isna, notna
from pandas.core.dtypes.common import (
    is_bool_dtype,
    is_categorical_dtype,
    is_object_dtype,
    is_string_like,
    is_list_like,
    is_scalar,
    is_integer,
    is_re)

import pandas.core.common as com
from pandas.core.algorithms import take_1d
import pandas.compat as compat
from pandas.core.base import NoNewAttributesMixin
from pandas.util._decorators import Appender
import re
import pandas._libs.lib as lib
import pandas._libs.ops as libops
import warnings
import textwrap
import codecs

_cpython_optimized_encoders = (
    "utf-8", "utf8", "latin-1", "latin1", "iso-8859-1", "mbcs", "ascii"
)
_cpython_optimized_decoders = _cpython_optimized_encoders + (
    "utf-16", "utf-32"
)

_shared_docs = dict()


def _get_array_list(arr, others):
    """
    Auxiliary function for :func:`str_cat`

    Parameters
    ----------
    arr : ndarray
        The left-most ndarray of the concatenation
    others : list, ndarray, Series
        The rest of the content to concatenate. If list of list-likes,
        all elements must be passable to ``np.asarray``.

    Returns
    -------
    list
        List of all necessary arrays
    """
    from pandas.core.series import Series

    if len(others) and isinstance(com._values_from_object(others)[0],
                                  (list, np.ndarray, Series)):
        arrays = [arr] + list(others)
    else:
        arrays = [arr, others]

    return [np.asarray(x, dtype=object) for x in arrays]


def str_cat(arr, others=None, sep=None, na_rep=None):
    """
    Auxiliary function for :meth:`str.cat`

    If `others` is specified, this function concatenates the Series/Index
    and elements of `others` element-wise.
    If `others` is not being passed then all values in the Series are
    concatenated in a single string with a given `sep`.

    Parameters
    ----------
    others : list-like, or list of list-likes, optional
        List-likes (or a list of them) of the same length as calling object.
        If None, returns str concatenating strings of the Series.
    sep : string or None, default None
        If None, concatenates without any separator.
    na_rep : string or None, default None
        If None, NA in the series are ignored.

    Returns
    -------
    concat
        ndarray containing concatenated results (if `others is not None`)
        or str (if `others is None`)
    """
    if sep is None:
        sep = ''

    if others is not None:
        arrays = _get_array_list(arr, others)

        n = _length_check(arrays)
        masks = np.array([isna(x) for x in arrays])
        cats = None

        if na_rep is None:
            na_mask = np.logical_or.reduce(masks, axis=0)

            result = np.empty(n, dtype=object)
            np.putmask(result, na_mask, np.nan)

            notmask = ~na_mask

            tuples = zip(*[x[notmask] for x in arrays])
            cats = [sep.join(tup) for tup in tuples]

            result[notmask] = cats
        else:
            for i, x in enumerate(arrays):
                x = np.where(masks[i], na_rep, x)
                if cats is None:
                    cats = x
                else:
                    cats = cats + sep + x

            result = cats

        return result
    else:
        arr = np.asarray(arr, dtype=object)
        mask = isna(arr)
        if na_rep is None and mask.any():
            if sep == '':
                na_rep = ''
            else:
                return sep.join(arr[notna(arr)])
        return sep.join(np.where(mask, na_rep, arr))


def _length_check(others):
    n = None
    for x in others:
        try:
            if n is None:
                n = len(x)
            elif len(x) != n:
                raise ValueError('All arrays must be same length')
        except TypeError:
            raise ValueError('Must pass arrays containing strings to str_cat')
    return n


def _na_map(f, arr, na_result=np.nan, dtype=object):
    # should really _check_ for NA
    return _map(f, arr, na_mask=True, na_value=na_result, dtype=dtype)


def _map(f, arr, na_mask=False, na_value=np.nan, dtype=object):
    if not len(arr):
        return np.ndarray(0, dtype=dtype)

    if isinstance(arr, ABCSeries):
        arr = arr.values
    if not isinstance(arr, np.ndarray):
        arr = np.asarray(arr, dtype=object)
    if na_mask:
        mask = isna(arr)
        try:
            convert = not all(mask)
            result = lib.map_infer_mask(arr, f, mask.view(np.uint8), convert)
        except (TypeError, AttributeError) as e:
            # Reraise the exception if callable `f` got wrong number of args.
            # The user may want to be warned by this, instead of getting NaN
            if compat.PY2:
                p_err = r'takes (no|(exactly|at (least|most)) ?\d+) arguments?'
            else:
                p_err = (r'((takes)|(missing)) (?(2)from \d+ to )?\d+ '
                         r'(?(3)required )positional arguments?')

            if len(e.args) >= 1 and re.search(p_err, e.args[0]):
                raise e

            def g(x):
                try:
                    return f(x)
                except (TypeError, AttributeError):
                    return na_value

            return _map(g, arr, dtype=dtype)
        if na_value is not np.nan:
            np.putmask(result, mask, na_value)
            if result.dtype == object:
                result = lib.maybe_convert_objects(result)
        return result
    else:
        return lib.map_infer(arr, f)


def str_count(arr, pat, flags=0):
    """
    Count occurrences of pattern in each string of the Series/Index.

    This function is used to count the number of times a particular regex
    pattern is repeated in each of the string elements of the
    :class:`~pandas.Series`.

    Parameters
    ----------
    pat : str
        Valid regular expression.
    flags : int, default 0, meaning no flags
        Flags for the `re` module. For a complete list, `see here
        <https://docs.python.org/3/howto/regex.html#compilation-flags>`_.
    **kwargs
        For compatibility with other string methods. Not used.

    Returns
    -------
    counts : Series or Index
        Same type as the calling object containing the integer counts.

    Notes
    -----
    Some characters need to be escaped when passing in `pat`.
    eg. ``'$'`` has a special meaning in regex and must be escaped when
    finding this literal character.

    See Also
    --------
    re : Standard library module for regular expressions.
    str.count : Standard library version, without regular expression support.

    Examples
    --------
    >>> s = pd.Series(['A', 'B', 'Aaba', 'Baca', np.nan, 'CABA', 'cat'])
    >>> s.str.count('a')
    0    0.0
    1    0.0
    2    2.0
    3    2.0
    4    NaN
    5    0.0
    6    1.0
    dtype: float64

    Escape ``'$'`` to find the literal dollar sign.

    >>> s = pd.Series(['$', 'B', 'Aab$', '$$ca', 'C$B$', 'cat'])
    >>> s.str.count('\\$')
    0    1
    1    0
    2    1
    3    2
    4    2
    5    0
    dtype: int64

    This is also available on Index

    >>> pd.Index(['A', 'A', 'Aaba', 'cat']).str.count('a')
    Int64Index([0, 0, 2, 1], dtype='int64')
    """
    regex = re.compile(pat, flags=flags)
    f = lambda x: len(regex.findall(x))
    return _na_map(f, arr, dtype=int)


def str_contains(arr, pat, case=True, flags=0, na=np.nan, regex=True):
    """
    Test if pattern or regex is contained within a string of a Series or Index.

    Return boolean Series or Index based on whether a given pattern or regex is
    contained within a string of a Series or Index.

    Parameters
    ----------
    pat : str
        Character sequence or regular expression.
    case : bool, default True
        If True, case sensitive.
    flags : int, default 0 (no flags)
        Flags to pass through to the re module, e.g. re.IGNORECASE.
    na : default NaN
        Fill value for missing values.
    regex : bool, default True
        If True, assumes the pat is a regular expression.

        If False, treats the pat as a literal string.

    Returns
    -------
    Series or Index of boolean values
        A Series or Index of boolean values indicating whether the
        given pattern is contained within the string of each element
        of the Series or Index.

    See Also
    --------
    match : analogous, but stricter, relying on re.match instead of re.search

    Examples
    --------

    Returning a Series of booleans using only a literal pattern.

    >>> s1 = pd.Series(['Mouse', 'dog', 'house and parrot', '23', np.NaN])
    >>> s1.str.contains('og', regex=False)
    0    False
    1     True
    2    False
    3    False
    4      NaN
    dtype: object

    Returning an Index of booleans using only a literal pattern.

    >>> ind = pd.Index(['Mouse', 'dog', 'house and parrot', '23.0', np.NaN])
    >>> ind.str.contains('23', regex=False)
    Index([False, False, False, True, nan], dtype='object')

    Specifying case sensitivity using `case`.

    >>> s1.str.contains('oG', case=True, regex=True)
    0    False
    1    False
    2    False
    3    False
    4      NaN
    dtype: object

    Specifying `na` to be `False` instead of `NaN` replaces NaN values
    with `False`. If Series or Index does not contain NaN values
    the resultant dtype will be `bool`, otherwise, an `object` dtype.

    >>> s1.str.contains('og', na=False, regex=True)
    0    False
    1     True
    2    False
    3    False
    4    False
    dtype: bool

    Returning 'house' or 'dog' when either expression occurs in a string.

    >>> s1.str.contains('house|dog', regex=True)
    0    False
    1     True
    2     True
    3    False
    4      NaN
    dtype: object

    Ignoring case sensitivity using `flags` with regex.

    >>> import re
    >>> s1.str.contains('PARROT', flags=re.IGNORECASE, regex=True)
    0    False
    1    False
    2     True
    3    False
    4      NaN
    dtype: object

    Returning any digit using regular expression.

    >>> s1.str.contains('\\d', regex=True)
    0    False
    1    False
    2    False
    3     True
    4      NaN
    dtype: object

    Ensure `pat` is a not a literal pattern when `regex` is set to True.
    Note in the following example one might expect only `s2[1]` and `s2[3]` to
    return `True`. However, '.0' as a regex matches any character
    followed by a 0.

    >>> s2 = pd.Series(['40','40.0','41','41.0','35'])
    >>> s2.str.contains('.0', regex=True)
    0     True
    1     True
    2    False
    3     True
    4    False
    dtype: bool
    """
    if regex:
        if not case:
            flags |= re.IGNORECASE

        regex = re.compile(pat, flags=flags)

        if regex.groups > 0:
            warnings.warn("This pattern has match groups. To actually get the"
                          " groups, use str.extract.", UserWarning,
                          stacklevel=3)

        f = lambda x: bool(regex.search(x))
    else:
        if case:
            f = lambda x: pat in x
        else:
            upper_pat = pat.upper()
            f = lambda x: upper_pat in x
            uppered = _na_map(lambda x: x.upper(), arr)
            return _na_map(f, uppered, na, dtype=bool)
    return _na_map(f, arr, na, dtype=bool)


def str_startswith(arr, pat, na=np.nan):
    """
    Test if the start of each string element matches a pattern.

    Equivalent to :meth:`str.startswith`.

    Parameters
    ----------
    pat : str
        Character sequence. Regular expressions are not accepted.
    na : object, default NaN
        Object shown if element tested is not a string.

    Returns
    -------
    Series or Index of bool
        A Series of booleans indicating whether the given pattern matches
        the start of each string element.

    See Also
    --------
    str.startswith : Python standard library string method.
    Series.str.endswith : Same as startswith, but tests the end of string.
    Series.str.contains : Tests if string element contains a pattern.

    Examples
    --------
    >>> s = pd.Series(['bat', 'Bear', 'cat', np.nan])
    >>> s
    0     bat
    1    Bear
    2     cat
    3     NaN
    dtype: object

    >>> s.str.startswith('b')
    0     True
    1    False
    2    False
    3      NaN
    dtype: object

    Specifying `na` to be `False` instead of `NaN`.

    >>> s.str.startswith('b', na=False)
    0     True
    1    False
    2    False
    3    False
    dtype: bool
    """
    f = lambda x: x.startswith(pat)
    return _na_map(f, arr, na, dtype=bool)


def str_endswith(arr, pat, na=np.nan):
    """
    Test if the end of each string element matches a pattern.

    Equivalent to :meth:`str.endswith`.

    Parameters
    ----------
    pat : str
        Character sequence. Regular expressions are not accepted.
    na : object, default NaN
        Object shown if element tested is not a string.

    Returns
    -------
    Series or Index of bool
        A Series of booleans indicating whether the given pattern matches
        the end of each string element.

    See Also
    --------
    str.endswith : Python standard library string method.
    Series.str.startswith : Same as endswith, but tests the start of string.
    Series.str.contains : Tests if string element contains a pattern.

    Examples
    --------
    >>> s = pd.Series(['bat', 'bear', 'caT', np.nan])
    >>> s
    0     bat
    1    bear
    2     caT
    3     NaN
    dtype: object

    >>> s.str.endswith('t')
    0     True
    1    False
    2    False
    3      NaN
    dtype: object

    Specifying `na` to be `False` instead of `NaN`.

    >>> s.str.endswith('t', na=False)
    0     True
    1    False
    2    False
    3    False
    dtype: bool
    """
    f = lambda x: x.endswith(pat)
    return _na_map(f, arr, na, dtype=bool)


def str_replace(arr, pat, repl, n=-1, case=None, flags=0, regex=True):
    r"""
    Replace occurrences of pattern/regex in the Series/Index with
    some other string. Equivalent to :meth:`str.replace` or
    :func:`re.sub`.

    Parameters
    ----------
    pat : string or compiled regex
        String can be a character sequence or regular expression.

        .. versionadded:: 0.20.0
            `pat` also accepts a compiled regex.

    repl : string or callable
        Replacement string or a callable. The callable is passed the regex
        match object and must return a replacement string to be used.
        See :func:`re.sub`.

        .. versionadded:: 0.20.0
            `repl` also accepts a callable.

    n : int, default -1 (all)
        Number of replacements to make from start
    case : boolean, default None
        - If True, case sensitive (the default if `pat` is a string)
        - Set to False for case insensitive
        - Cannot be set if `pat` is a compiled regex
    flags : int, default 0 (no flags)
        - re module flags, e.g. re.IGNORECASE
        - Cannot be set if `pat` is a compiled regex
    regex : boolean, default True
        - If True, assumes the passed-in pattern is a regular expression.
        - If False, treats the pattern as a literal string
        - Cannot be set to False if `pat` is a compiled regex or `repl` is
          a callable.

        .. versionadded:: 0.23.0

    Returns
    -------
    replaced : Series/Index of objects

    Raises
    ------
    ValueError
        * if `regex` is False and `repl` is a callable or `pat` is a compiled
          regex
        * if `pat` is a compiled regex and `case` or `flags` is set

    Notes
    -----
    When `pat` is a compiled regex, all flags should be included in the
    compiled regex. Use of `case`, `flags`, or `regex=False` with a compiled
    regex will raise an error.

    Examples
    --------
    When `pat` is a string and `regex` is True (the default), the given `pat`
    is compiled as a regex. When `repl` is a string, it replaces matching
    regex patterns as with :meth:`re.sub`. NaN value(s) in the Series are
    left as is:

    >>> pd.Series(['foo', 'fuz', np.nan]).str.replace('f.', 'ba', regex=True)
    0    bao
    1    baz
    2    NaN
    dtype: object

    When `pat` is a string and `regex` is False, every `pat` is replaced with
    `repl` as with :meth:`str.replace`:

    >>> pd.Series(['f.o', 'fuz', np.nan]).str.replace('f.', 'ba', regex=False)
    0    bao
    1    fuz
    2    NaN
    dtype: object

    When `repl` is a callable, it is called on every `pat` using
    :func:`re.sub`. The callable should expect one positional argument
    (a regex object) and return a string.

    To get the idea:

    >>> pd.Series(['foo', 'fuz', np.nan]).str.replace('f', repr)
    0    <_sre.SRE_Match object; span=(0, 1), match='f'>oo
    1    <_sre.SRE_Match object; span=(0, 1), match='f'>uz
    2                                                  NaN
    dtype: object

    Reverse every lowercase alphabetic word:

    >>> repl = lambda m: m.group(0)[::-1]
    >>> pd.Series(['foo 123', 'bar baz', np.nan]).str.replace(r'[a-z]+', repl)
    0    oof 123
    1    rab zab
    2        NaN
    dtype: object

    Using regex groups (extract second group and swap case):

    >>> pat = r"(?P<one>\w+) (?P<two>\w+) (?P<three>\w+)"
    >>> repl = lambda m: m.group('two').swapcase()
    >>> pd.Series(['One Two Three', 'Foo Bar Baz']).str.replace(pat, repl)
    0    tWO
    1    bAR
    dtype: object

    Using a compiled regex with flags

    >>> regex_pat = re.compile(r'FUZ', flags=re.IGNORECASE)
    >>> pd.Series(['foo', 'fuz', np.nan]).str.replace(regex_pat, 'bar')
    0    foo
    1    bar
    2    NaN
    dtype: object

    """

    # Check whether repl is valid (GH 13438, GH 15055)
    if not (is_string_like(repl) or callable(repl)):
        raise TypeError("repl must be a string or callable")

    is_compiled_re = is_re(pat)
    if regex:
        if is_compiled_re:
            if (case is not None) or (flags != 0):
                raise ValueError("case and flags cannot be set"
                                 " when pat is a compiled regex")
        else:
            # not a compiled regex
            # set default case
            if case is None:
                case = True

            # add case flag, if provided
            if case is False:
                flags |= re.IGNORECASE
        if is_compiled_re or len(pat) > 1 or flags or callable(repl):
            n = n if n >= 0 else 0
            compiled = re.compile(pat, flags=flags)
            f = lambda x: compiled.sub(repl=repl, string=x, count=n)
        else:
            f = lambda x: x.replace(pat, repl, n)
    else:
        if is_compiled_re:
            raise ValueError("Cannot use a compiled regex as replacement "
                             "pattern with regex=False")
        if callable(repl):
            raise ValueError("Cannot use a callable replacement when "
                             "regex=False")
        f = lambda x: x.replace(pat, repl, n)

    return _na_map(f, arr)


def str_repeat(arr, repeats):
    """
    Duplicate each string in the Series/Index by indicated number
    of times.

    Parameters
    ----------
    repeats : int or array
        Same value for all (int) or different value per (array)

    Returns
    -------
    repeated : Series/Index of objects
    """
    if is_scalar(repeats):

        def rep(x):
            try:
                return compat.binary_type.__mul__(x, repeats)
            except TypeError:
                return compat.text_type.__mul__(x, repeats)

        return _na_map(rep, arr)
    else:

        def rep(x, r):
            try:
                return compat.binary_type.__mul__(x, r)
            except TypeError:
                return compat.text_type.__mul__(x, r)

        repeats = np.asarray(repeats, dtype=object)
        result = libops.vec_binop(com._values_from_object(arr), repeats, rep)
        return result


def str_match(arr, pat, case=True, flags=0, na=np.nan, as_indexer=None):
    """
    Determine if each string matches a regular expression.

    Parameters
    ----------
    pat : string
        Character sequence or regular expression
    case : boolean, default True
        If True, case sensitive
    flags : int, default 0 (no flags)
        re module flags, e.g. re.IGNORECASE
    na : default NaN, fill value for missing values.
    as_indexer
        .. deprecated:: 0.21.0

    Returns
    -------
    Series/array of boolean values

    See Also
    --------
    contains : analogous, but less strict, relying on re.search instead of
        re.match
    extract : extract matched groups

    """
    if not case:
        flags |= re.IGNORECASE

    regex = re.compile(pat, flags=flags)

    if (as_indexer is False) and (regex.groups > 0):
        raise ValueError("as_indexer=False with a pattern with groups is no "
                         "longer supported. Use '.str.extract(pat)' instead")
    elif as_indexer is not None:
        # Previously, this keyword was used for changing the default but
        # deprecated behaviour. This keyword is now no longer needed.
        warnings.warn("'as_indexer' keyword was specified but is ignored "
                      "(match now returns a boolean indexer by default), "
                      "and will be removed in a future version.",
                      FutureWarning, stacklevel=3)

    dtype = bool
    f = lambda x: bool(regex.match(x))

    return _na_map(f, arr, na, dtype=dtype)


def _get_single_group_name(rx):
    try:
        return list(rx.groupindex.keys()).pop()
    except IndexError:
        return None


def _groups_or_na_fun(regex):
    """Used in both extract_noexpand and extract_frame"""
    if regex.groups == 0:
        raise ValueError("pattern contains no capture groups")
    empty_row = [np.nan] * regex.groups

    def f(x):
        if not isinstance(x, compat.string_types):
            return empty_row
        m = regex.search(x)
        if m:
            return [np.nan if item is None else item for item in m.groups()]
        else:
            return empty_row
    return f


def _str_extract_noexpand(arr, pat, flags=0):
    """
    Find groups in each string in the Series using passed regular
    expression. This function is called from
    str_extract(expand=False), and can return Series, DataFrame, or
    Index.

    """
    from pandas import DataFrame, Index

    regex = re.compile(pat, flags=flags)
    groups_or_na = _groups_or_na_fun(regex)

    if regex.groups == 1:
        result = np.array([groups_or_na(val)[0] for val in arr], dtype=object)
        name = _get_single_group_name(regex)
    else:
        if isinstance(arr, Index):
            raise ValueError("only one regex group is supported with Index")
        name = None
        names = dict(zip(regex.groupindex.values(), regex.groupindex.keys()))
        columns = [names.get(1 + i, i) for i in range(regex.groups)]
        if arr.empty:
            result = DataFrame(columns=columns, dtype=object)
        else:
            result = DataFrame(
                [groups_or_na(val) for val in arr],
                columns=columns,
                index=arr.index,
                dtype=object)
    return result, name


def _str_extract_frame(arr, pat, flags=0):
    """
    For each subject string in the Series, extract groups from the
    first match of regular expression pat. This function is called from
    str_extract(expand=True), and always returns a DataFrame.

    """
    from pandas import DataFrame

    regex = re.compile(pat, flags=flags)
    groups_or_na = _groups_or_na_fun(regex)
    names = dict(zip(regex.groupindex.values(), regex.groupindex.keys()))
    columns = [names.get(1 + i, i) for i in range(regex.groups)]

    if len(arr) == 0:
        return DataFrame(columns=columns, dtype=object)
    try:
        result_index = arr.index
    except AttributeError:
        result_index = None
    return DataFrame(
        [groups_or_na(val) for val in arr],
        columns=columns,
        index=result_index,
        dtype=object)


def str_extract(arr, pat, flags=0, expand=True):
    r"""
    For each subject string in the Series, extract groups from the
    first match of regular expression pat.

    Parameters
    ----------
    pat : string
        Regular expression pattern with capturing groups
    flags : int, default 0 (no flags)
        re module flags, e.g. re.IGNORECASE

    expand : bool, default True
        * If True, return DataFrame.
        * If False, return Series/Index/DataFrame.

        .. versionadded:: 0.18.0

    Returns
    -------
    DataFrame with one row for each subject string, and one column for
    each group. Any capture group names in regular expression pat will
    be used for column names; otherwise capture group numbers will be
    used. The dtype of each result column is always object, even when
    no match is found. If expand=False and pat has only one capture group,
    then return a Series (if subject is a Series) or Index (if subject
    is an Index).

    See Also
    --------
    extractall : returns all matches (not just the first match)

    Examples
    --------
    A pattern with two groups will return a DataFrame with two columns.
    Non-matches will be NaN.

    >>> s = Series(['a1', 'b2', 'c3'])
    >>> s.str.extract(r'([ab])(\d)')
         0    1
    0    a    1
    1    b    2
    2  NaN  NaN

    A pattern may contain optional groups.

    >>> s.str.extract(r'([ab])?(\d)')
         0  1
    0    a  1
    1    b  2
    2  NaN  3

    Named groups will become column names in the result.

    >>> s.str.extract(r'(?P<letter>[ab])(?P<digit>\d)')
      letter digit
    0      a     1
    1      b     2
    2    NaN   NaN

    A pattern with one group will return a DataFrame with one column
    if expand=True.

    >>> s.str.extract(r'[ab](\d)', expand=True)
         0
    0    1
    1    2
    2  NaN

    A pattern with one group will return a Series if expand=False.

    >>> s.str.extract(r'[ab](\d)', expand=False)
    0      1
    1      2
    2    NaN
    dtype: object

    """
    if not isinstance(expand, bool):
        raise ValueError("expand must be True or False")
    if expand:
        return _str_extract_frame(arr._orig, pat, flags=flags)
    else:
        result, name = _str_extract_noexpand(arr._data, pat, flags=flags)
        return arr._wrap_result(result, name=name, expand=expand)


def str_extractall(arr, pat, flags=0):
    r"""
    For each subject string in the Series, extract groups from all
    matches of regular expression pat. When each subject string in the
    Series has exactly one match, extractall(pat).xs(0, level='match')
    is the same as extract(pat).

    .. versionadded:: 0.18.0

    Parameters
    ----------
    pat : string
        Regular expression pattern with capturing groups
    flags : int, default 0 (no flags)
        re module flags, e.g. re.IGNORECASE

    Returns
    -------
    A DataFrame with one row for each match, and one column for each<commit_msg>DOC: Improved the docstring of str.extract() (Delhi) (#20141)<commit_after>import numpy as np

from pandas.compat import zip
from pandas.core.dtypes.generic import ABCSeries, ABCIndex
from pandas.core.dtypes.missing import isna, notna
from pandas.core.dtypes.common import (
    is_bool_dtype,
    is_categorical_dtype,
    is_object_dtype,
    is_string_like,
    is_list_like,
    is_scalar,
    is_integer,
    is_re)

import pandas.core.common as com
from pandas.core.algorithms import take_1d
import pandas.compat as compat
from pandas.core.base import NoNewAttributesMixin
from pandas.util._decorators import Appender
import re
import pandas._libs.lib as lib
import pandas._libs.ops as libops
import warnings
import textwrap
import codecs

_cpython_optimized_encoders = (
    "utf-8", "utf8", "latin-1", "latin1", "iso-8859-1", "mbcs", "ascii"
)
_cpython_optimized_decoders = _cpython_optimized_encoders + (
    "utf-16", "utf-32"
)

_shared_docs = dict()


def _get_array_list(arr, others):
    """
    Auxiliary function for :func:`str_cat`

    Parameters
    ----------
    arr : ndarray
        The left-most ndarray of the concatenation
    others : list, ndarray, Series
        The rest of the content to concatenate. If list of list-likes,
        all elements must be passable to ``np.asarray``.

    Returns
    -------
    list
        List of all necessary arrays
    """
    from pandas.core.series import Series

    if len(others) and isinstance(com._values_from_object(others)[0],
                                  (list, np.ndarray, Series)):
        arrays = [arr] + list(others)
    else:
        arrays = [arr, others]

    return [np.asarray(x, dtype=object) for x in arrays]


def str_cat(arr, others=None, sep=None, na_rep=None):
    """
    Auxiliary function for :meth:`str.cat`

    If `others` is specified, this function concatenates the Series/Index
    and elements of `others` element-wise.
    If `others` is not being passed then all values in the Series are
    concatenated in a single string with a given `sep`.

    Parameters
    ----------
    others : list-like, or list of list-likes, optional
        List-likes (or a list of them) of the same length as calling object.
        If None, returns str concatenating strings of the Series.
    sep : string or None, default None
        If None, concatenates without any separator.
    na_rep : string or None, default None
        If None, NA in the series are ignored.

    Returns
    -------
    concat
        ndarray containing concatenated results (if `others is not None`)
        or str (if `others is None`)
    """
    if sep is None:
        sep = ''

    if others is not None:
        arrays = _get_array_list(arr, others)

        n = _length_check(arrays)
        masks = np.array([isna(x) for x in arrays])
        cats = None

        if na_rep is None:
            na_mask = np.logical_or.reduce(masks, axis=0)

            result = np.empty(n, dtype=object)
            np.putmask(result, na_mask, np.nan)

            notmask = ~na_mask

            tuples = zip(*[x[notmask] for x in arrays])
            cats = [sep.join(tup) for tup in tuples]

            result[notmask] = cats
        else:
            for i, x in enumerate(arrays):
                x = np.where(masks[i], na_rep, x)
                if cats is None:
                    cats = x
                else:
                    cats = cats + sep + x

            result = cats

        return result
    else:
        arr = np.asarray(arr, dtype=object)
        mask = isna(arr)
        if na_rep is None and mask.any():
            if sep == '':
                na_rep = ''
            else:
                return sep.join(arr[notna(arr)])
        return sep.join(np.where(mask, na_rep, arr))


def _length_check(others):
    n = None
    for x in others:
        try:
            if n is None:
                n = len(x)
            elif len(x) != n:
                raise ValueError('All arrays must be same length')
        except TypeError:
            raise ValueError('Must pass arrays containing strings to str_cat')
    return n


def _na_map(f, arr, na_result=np.nan, dtype=object):
    # should really _check_ for NA
    return _map(f, arr, na_mask=True, na_value=na_result, dtype=dtype)


def _map(f, arr, na_mask=False, na_value=np.nan, dtype=object):
    if not len(arr):
        return np.ndarray(0, dtype=dtype)

    if isinstance(arr, ABCSeries):
        arr = arr.values
    if not isinstance(arr, np.ndarray):
        arr = np.asarray(arr, dtype=object)
    if na_mask:
        mask = isna(arr)
        try:
            convert = not all(mask)
            result = lib.map_infer_mask(arr, f, mask.view(np.uint8), convert)
        except (TypeError, AttributeError) as e:
            # Reraise the exception if callable `f` got wrong number of args.
            # The user may want to be warned by this, instead of getting NaN
            if compat.PY2:
                p_err = r'takes (no|(exactly|at (least|most)) ?\d+) arguments?'
            else:
                p_err = (r'((takes)|(missing)) (?(2)from \d+ to )?\d+ '
                         r'(?(3)required )positional arguments?')

            if len(e.args) >= 1 and re.search(p_err, e.args[0]):
                raise e

            def g(x):
                try:
                    return f(x)
                except (TypeError, AttributeError):
                    return na_value

            return _map(g, arr, dtype=dtype)
        if na_value is not np.nan:
            np.putmask(result, mask, na_value)
            if result.dtype == object:
                result = lib.maybe_convert_objects(result)
        return result
    else:
        return lib.map_infer(arr, f)


def str_count(arr, pat, flags=0):
    """
    Count occurrences of pattern in each string of the Series/Index.

    This function is used to count the number of times a particular regex
    pattern is repeated in each of the string elements of the
    :class:`~pandas.Series`.

    Parameters
    ----------
    pat : str
        Valid regular expression.
    flags : int, default 0, meaning no flags
        Flags for the `re` module. For a complete list, `see here
        <https://docs.python.org/3/howto/regex.html#compilation-flags>`_.
    **kwargs
        For compatibility with other string methods. Not used.

    Returns
    -------
    counts : Series or Index
        Same type as the calling object containing the integer counts.

    Notes
    -----
    Some characters need to be escaped when passing in `pat`.
    eg. ``'$'`` has a special meaning in regex and must be escaped when
    finding this literal character.

    See Also
    --------
    re : Standard library module for regular expressions.
    str.count : Standard library version, without regular expression support.

    Examples
    --------
    >>> s = pd.Series(['A', 'B', 'Aaba', 'Baca', np.nan, 'CABA', 'cat'])
    >>> s.str.count('a')
    0    0.0
    1    0.0
    2    2.0
    3    2.0
    4    NaN
    5    0.0
    6    1.0
    dtype: float64

    Escape ``'$'`` to find the literal dollar sign.

    >>> s = pd.Series(['$', 'B', 'Aab$', '$$ca', 'C$B$', 'cat'])
    >>> s.str.count('\\$')
    0    1
    1    0
    2    1
    3    2
    4    2
    5    0
    dtype: int64

    This is also available on Index

    >>> pd.Index(['A', 'A', 'Aaba', 'cat']).str.count('a')
    Int64Index([0, 0, 2, 1], dtype='int64')
    """
    regex = re.compile(pat, flags=flags)
    f = lambda x: len(regex.findall(x))
    return _na_map(f, arr, dtype=int)


def str_contains(arr, pat, case=True, flags=0, na=np.nan, regex=True):
    """
    Test if pattern or regex is contained within a string of a Series or Index.

    Return boolean Series or Index based on whether a given pattern or regex is
    contained within a string of a Series or Index.

    Parameters
    ----------
    pat : str
        Character sequence or regular expression.
    case : bool, default True
        If True, case sensitive.
    flags : int, default 0 (no flags)
        Flags to pass through to the re module, e.g. re.IGNORECASE.
    na : default NaN
        Fill value for missing values.
    regex : bool, default True
        If True, assumes the pat is a regular expression.

        If False, treats the pat as a literal string.

    Returns
    -------
    Series or Index of boolean values
        A Series or Index of boolean values indicating whether the
        given pattern is contained within the string of each element
        of the Series or Index.

    See Also
    --------
    match : analogous, but stricter, relying on re.match instead of re.search

    Examples
    --------

    Returning a Series of booleans using only a literal pattern.

    >>> s1 = pd.Series(['Mouse', 'dog', 'house and parrot', '23', np.NaN])
    >>> s1.str.contains('og', regex=False)
    0    False
    1     True
    2    False
    3    False
    4      NaN
    dtype: object

    Returning an Index of booleans using only a literal pattern.

    >>> ind = pd.Index(['Mouse', 'dog', 'house and parrot', '23.0', np.NaN])
    >>> ind.str.contains('23', regex=False)
    Index([False, False, False, True, nan], dtype='object')

    Specifying case sensitivity using `case`.

    >>> s1.str.contains('oG', case=True, regex=True)
    0    False
    1    False
    2    False
    3    False
    4      NaN
    dtype: object

    Specifying `na` to be `False` instead of `NaN` replaces NaN values
    with `False`. If Series or Index does not contain NaN values
    the resultant dtype will be `bool`, otherwise, an `object` dtype.

    >>> s1.str.contains('og', na=False, regex=True)
    0    False
    1     True
    2    False
    3    False
    4    False
    dtype: bool

    Returning 'house' or 'dog' when either expression occurs in a string.

    >>> s1.str.contains('house|dog', regex=True)
    0    False
    1     True
    2     True
    3    False
    4      NaN
    dtype: object

    Ignoring case sensitivity using `flags` with regex.

    >>> import re
    >>> s1.str.contains('PARROT', flags=re.IGNORECASE, regex=True)
    0    False
    1    False
    2     True
    3    False
    4      NaN
    dtype: object

    Returning any digit using regular expression.

    >>> s1.str.contains('\\d', regex=True)
    0    False
    1    False
    2    False
    3     True
    4      NaN
    dtype: object

    Ensure `pat` is a not a literal pattern when `regex` is set to True.
    Note in the following example one might expect only `s2[1]` and `s2[3]` to
    return `True`. However, '.0' as a regex matches any character
    followed by a 0.

    >>> s2 = pd.Series(['40','40.0','41','41.0','35'])
    >>> s2.str.contains('.0', regex=True)
    0     True
    1     True
    2    False
    3     True
    4    False
    dtype: bool
    """
    if regex:
        if not case:
            flags |= re.IGNORECASE

        regex = re.compile(pat, flags=flags)

        if regex.groups > 0:
            warnings.warn("This pattern has match groups. To actually get the"
                          " groups, use str.extract.", UserWarning,
                          stacklevel=3)

        f = lambda x: bool(regex.search(x))
    else:
        if case:
            f = lambda x: pat in x
        else:
            upper_pat = pat.upper()
            f = lambda x: upper_pat in x
            uppered = _na_map(lambda x: x.upper(), arr)
            return _na_map(f, uppered, na, dtype=bool)
    return _na_map(f, arr, na, dtype=bool)


def str_startswith(arr, pat, na=np.nan):
    """
    Test if the start of each string element matches a pattern.

    Equivalent to :meth:`str.startswith`.

    Parameters
    ----------
    pat : str
        Character sequence. Regular expressions are not accepted.
    na : object, default NaN
        Object shown if element tested is not a string.

    Returns
    -------
    Series or Index of bool
        A Series of booleans indicating whether the given pattern matches
        the start of each string element.

    See Also
    --------
    str.startswith : Python standard library string method.
    Series.str.endswith : Same as startswith, but tests the end of string.
    Series.str.contains : Tests if string element contains a pattern.

    Examples
    --------
    >>> s = pd.Series(['bat', 'Bear', 'cat', np.nan])
    >>> s
    0     bat
    1    Bear
    2     cat
    3     NaN
    dtype: object

    >>> s.str.startswith('b')
    0     True
    1    False
    2    False
    3      NaN
    dtype: object

    Specifying `na` to be `False` instead of `NaN`.

    >>> s.str.startswith('b', na=False)
    0     True
    1    False
    2    False
    3    False
    dtype: bool
    """
    f = lambda x: x.startswith(pat)
    return _na_map(f, arr, na, dtype=bool)


def str_endswith(arr, pat, na=np.nan):
    """
    Test if the end of each string element matches a pattern.

    Equivalent to :meth:`str.endswith`.

    Parameters
    ----------
    pat : str
        Character sequence. Regular expressions are not accepted.
    na : object, default NaN
        Object shown if element tested is not a string.

    Returns
    -------
    Series or Index of bool
        A Series of booleans indicating whether the given pattern matches
        the end of each string element.

    See Also
    --------
    str.endswith : Python standard library string method.
    Series.str.startswith : Same as endswith, but tests the start of string.
    Series.str.contains : Tests if string element contains a pattern.

    Examples
    --------
    >>> s = pd.Series(['bat', 'bear', 'caT', np.nan])
    >>> s
    0     bat
    1    bear
    2     caT
    3     NaN
    dtype: object

    >>> s.str.endswith('t')
    0     True
    1    False
    2    False
    3      NaN
    dtype: object

    Specifying `na` to be `False` instead of `NaN`.

    >>> s.str.endswith('t', na=False)
    0     True
    1    False
    2    False
    3    False
    dtype: bool
    """
    f = lambda x: x.endswith(pat)
    return _na_map(f, arr, na, dtype=bool)


def str_replace(arr, pat, repl, n=-1, case=None, flags=0, regex=True):
    r"""
    Replace occurrences of pattern/regex in the Series/Index with
    some other string. Equivalent to :meth:`str.replace` or
    :func:`re.sub`.

    Parameters
    ----------
    pat : string or compiled regex
        String can be a character sequence or regular expression.

        .. versionadded:: 0.20.0
            `pat` also accepts a compiled regex.

    repl : string or callable
        Replacement string or a callable. The callable is passed the regex
        match object and must return a replacement string to be used.
        See :func:`re.sub`.

        .. versionadded:: 0.20.0
            `repl` also accepts a callable.

    n : int, default -1 (all)
        Number of replacements to make from start
    case : boolean, default None
        - If True, case sensitive (the default if `pat` is a string)
        - Set to False for case insensitive
        - Cannot be set if `pat` is a compiled regex
    flags : int, default 0 (no flags)
        - re module flags, e.g. re.IGNORECASE
        - Cannot be set if `pat` is a compiled regex
    regex : boolean, default True
        - If True, assumes the passed-in pattern is a regular expression.
        - If False, treats the pattern as a literal string
        - Cannot be set to False if `pat` is a compiled regex or `repl` is
          a callable.

        .. versionadded:: 0.23.0

    Returns
    -------
    replaced : Series/Index of objects

    Raises
    ------
    ValueError
        * if `regex` is False and `repl` is a callable or `pat` is a compiled
          regex
        * if `pat` is a compiled regex and `case` or `flags` is set

    Notes
    -----
    When `pat` is a compiled regex, all flags should be included in the
    compiled regex. Use of `case`, `flags`, or `regex=False` with a compiled
    regex will raise an error.

    Examples
    --------
    When `pat` is a string and `regex` is True (the default), the given `pat`
    is compiled as a regex. When `repl` is a string, it replaces matching
    regex patterns as with :meth:`re.sub`. NaN value(s) in the Series are
    left as is:

    >>> pd.Series(['foo', 'fuz', np.nan]).str.replace('f.', 'ba', regex=True)
    0    bao
    1    baz
    2    NaN
    dtype: object

    When `pat` is a string and `regex` is False, every `pat` is replaced with
    `repl` as with :meth:`str.replace`:

    >>> pd.Series(['f.o', 'fuz', np.nan]).str.replace('f.', 'ba', regex=False)
    0    bao
    1    fuz
    2    NaN
    dtype: object

    When `repl` is a callable, it is called on every `pat` using
    :func:`re.sub`. The callable should expect one positional argument
    (a regex object) and return a string.

    To get the idea:

    >>> pd.Series(['foo', 'fuz', np.nan]).str.replace('f', repr)
    0    <_sre.SRE_Match object; span=(0, 1), match='f'>oo
    1    <_sre.SRE_Match object; span=(0, 1), match='f'>uz
    2                                                  NaN
    dtype: object

    Reverse every lowercase alphabetic word:

    >>> repl = lambda m: m.group(0)[::-1]
    >>> pd.Series(['foo 123', 'bar baz', np.nan]).str.replace(r'[a-z]+', repl)
    0    oof 123
    1    rab zab
    2        NaN
    dtype: object

    Using regex groups (extract second group and swap case):

    >>> pat = r"(?P<one>\w+) (?P<two>\w+) (?P<three>\w+)"
    >>> repl = lambda m: m.group('two').swapcase()
    >>> pd.Series(['One Two Three', 'Foo Bar Baz']).str.replace(pat, repl)
    0    tWO
    1    bAR
    dtype: object

    Using a compiled regex with flags

    >>> regex_pat = re.compile(r'FUZ', flags=re.IGNORECASE)
    >>> pd.Series(['foo', 'fuz', np.nan]).str.replace(regex_pat, 'bar')
    0    foo
    1    bar
    2    NaN
    dtype: object

    """

    # Check whether repl is valid (GH 13438, GH 15055)
    if not (is_string_like(repl) or callable(repl)):
        raise TypeError("repl must be a string or callable")

    is_compiled_re = is_re(pat)
    if regex:
        if is_compiled_re:
            if (case is not None) or (flags != 0):
                raise ValueError("case and flags cannot be set"
                                 " when pat is a compiled regex")
        else:
            # not a compiled regex
            # set default case
            if case is None:
                case = True

            # add case flag, if provided
            if case is False:
                flags |= re.IGNORECASE
        if is_compiled_re or len(pat) > 1 or flags or callable(repl):
            n = n if n >= 0 else 0
            compiled = re.compile(pat, flags=flags)
            f = lambda x: compiled.sub(repl=repl, string=x, count=n)
        else:
            f = lambda x: x.replace(pat, repl, n)
    else:
        if is_compiled_re:
            raise ValueError("Cannot use a compiled regex as replacement "
                             "pattern with regex=False")
        if callable(repl):
            raise ValueError("Cannot use a callable replacement when "
                             "regex=False")
        f = lambda x: x.replace(pat, repl, n)

    return _na_map(f, arr)


def str_repeat(arr, repeats):
    """
    Duplicate each string in the Series/Index by indicated number
    of times.

    Parameters
    ----------
    repeats : int or array
        Same value for all (int) or different value per (array)

    Returns
    -------
    repeated : Series/Index of objects
    """
    if is_scalar(repeats):

        def rep(x):
            try:
                return compat.binary_type.__mul__(x, repeats)
            except TypeError:
                return compat.text_type.__mul__(x, repeats)

        return _na_map(rep, arr)
    else:

        def rep(x, r):
            try:
                return compat.binary_type.__mul__(x, r)
            except TypeError:
                return compat.text_type.__mul__(x, r)

        repeats = np.asarray(repeats, dtype=object)
        result = libops.vec_binop(com._values_from_object(arr), repeats, rep)
        return result


def str_match(arr, pat, case=True, flags=0, na=np.nan, as_indexer=None):
    """
    Determine if each string matches a regular expression.

    Parameters
    ----------
    pat : string
        Character sequence or regular expression
    case : boolean, default True
        If True, case sensitive
    flags : int, default 0 (no flags)
        re module flags, e.g. re.IGNORECASE
    na : default NaN, fill value for missing values.
    as_indexer
        .. deprecated:: 0.21.0

    Returns
    -------
    Series/array of boolean values

    See Also
    --------
    contains : analogous, but less strict, relying on re.search instead of
        re.match
    extract : extract matched groups

    """
    if not case:
        flags |= re.IGNORECASE

    regex = re.compile(pat, flags=flags)

    if (as_indexer is False) and (regex.groups > 0):
        raise ValueError("as_indexer=False with a pattern with groups is no "
                         "longer supported. Use '.str.extract(pat)' instead")
    elif as_indexer is not None:
        # Previously, this keyword was used for changing the default but
        # deprecated behaviour. This keyword is now no longer needed.
        warnings.warn("'as_indexer' keyword was specified but is ignored "
                      "(match now returns a boolean indexer by default), "
                      "and will be removed in a future version.",
                      FutureWarning, stacklevel=3)

    dtype = bool
    f = lambda x: bool(regex.match(x))

    return _na_map(f, arr, na, dtype=dtype)


def _get_single_group_name(rx):
    try:
        return list(rx.groupindex.keys()).pop()
    except IndexError:
        return None


def _groups_or_na_fun(regex):
    """Used in both extract_noexpand and extract_frame"""
    if regex.groups == 0:
        raise ValueError("pattern contains no capture groups")
    empty_row = [np.nan] * regex.groups

    def f(x):
        if not isinstance(x, compat.string_types):
            return empty_row
        m = regex.search(x)
        if m:
            return [np.nan if item is None else item for item in m.groups()]
        else:
            return empty_row
    return f


def _str_extract_noexpand(arr, pat, flags=0):
    """
    Find groups in each string in the Series using passed regular
    expression. This function is called from
    str_extract(expand=False), and can return Series, DataFrame, or
    Index.

    """
    from pandas import DataFrame, Index

    regex = re.compile(pat, flags=flags)
    groups_or_na = _groups_or_na_fun(regex)

    if regex.groups == 1:
        result = np.array([groups_or_na(val)[0] for val in arr], dtype=object)
        name = _get_single_group_name(regex)
    else:
        if isinstance(arr, Index):
            raise ValueError("only one regex group is supported with Index")
        name = None
        names = dict(zip(regex.groupindex.values(), regex.groupindex.keys()))
        columns = [names.get(1 + i, i) for i in range(regex.groups)]
        if arr.empty:
            result = DataFrame(columns=columns, dtype=object)
        else:
            result = DataFrame(
                [groups_or_na(val) for val in arr],
                columns=columns,
                index=arr.index,
                dtype=object)
    return result, name


def _str_extract_frame(arr, pat, flags=0):
    """
    For each subject string in the Series, extract groups from the
    first match of regular expression pat. This function is called from
    str_extract(expand=True), and always returns a DataFrame.

    """
    from pandas import DataFrame

    regex = re.compile(pat, flags=flags)
    groups_or_na = _groups_or_na_fun(regex)
    names = dict(zip(regex.groupindex.values(), regex.groupindex.keys()))
    columns = [names.get(1 + i, i) for i in range(regex.groups)]

    if len(arr) == 0:
        return DataFrame(columns=columns, dtype=object)
    try:
        result_index = arr.index
    except AttributeError:
        result_index = None
    return DataFrame(
        [groups_or_na(val) for val in arr],
        columns=columns,
        index=result_index,
        dtype=object)


def str_extract(arr, pat, flags=0, expand=True):
    r"""
    Extract capture groups in the regex `pat` as columns in a DataFrame.

    For each subject string in the Series, extract groups from the
    first match of regular expression `pat`.

    Parameters
    ----------
    pat : string
        Regular expression pattern with capturing groups.
    flags : int, default 0 (no flags)
        ``re`` module flags, e.g. ``re.IGNORECASE``.
        See :mod:`re`
    expand : bool, default True
        If True, return DataFrame with one column per capture group.
        If False, return a Series/Index if there is one capture group
        or DataFrame if there are multiple capture groups.

        .. versionadded:: 0.18.0

    Returns
    -------
    DataFrame with one row for each subject string, and one column for
    each group. Any capture group names in regular expression pat will
    be used for column names; otherwise capture group numbers will be
    used. The dtype of each result column is always object, even when
    no match is found. If expand=False and pat has only one capture group,
    then return a Series (if subject is a Series) or Index (if subject
    is an Index).

    See Also
    --------
    extractall : returns all matches (not just the first match)

    Examples
    --------
    A pattern with two groups will return a DataFrame with two columns.
    Non-matches will be NaN.

    >>> s = pd.Series(['a1', 'b2', 'c3'])
    >>> s.str.extract(r'([ab])(\d)')
         0    1
    0    a    1
    1    b    2
    2  NaN  NaN

    A pattern may contain optional groups.

    >>> s.str.extract(r'([ab])?(\d)')
         0  1
    0    a  1
    1    b  2
    2  NaN  3

    Named groups will become column names in the result.

    >>> s.str.extract(r'(?P<letter>[ab])(?P<digit>\d)')
      letter digit
    0      a     1
    1      b     2
    2    NaN   NaN

    A pattern with one group will return a DataFrame with one column
    if expand=True.

    >>> s.str.extract(r'[ab](\d)', expand=True)
         0
    0    1
    1    2
    2  NaN

    A pattern with one group will return a Series if expand=False.

    >>> s.str.extract(r'[ab](\d)', expand=False)
    0      1
    1      2
    2    NaN
    dtype: object
    """
    if not isinstance(expand, bool):
        raise ValueError("expand must be True or False")
    if expand:
        return _str_extract_frame(arr._orig, pat, flags=flags)
    else:
        result, name = _str_extract_noexpand(arr._data, pat, flags=flags)
        return arr._wrap_result(result, name=name, expand=expand)


def str_extractall(arr, pat, flags=0):
    r"""
    For each subject string in the Series, extract groups from all
    matches of regular expression pat. When each subject string in the
    Series has exactly one match, extractall(pat).xs(0, level='match')
    is the same as extract(pat).

    .. versionadded:: 0.18.0

    Parameters
    ----------
    pat : string
        Regular expression pattern with capturing groups
    flags : int, default 0 (no flags)
        re module flags, e.g. re.IGNORECASE

    Returns
    -------
    A DataFrame with one row for each match, and one column for each
{'fraction_non_alphanumeric': 0.10381483853261367, 'fraction_numerical': 0.010666910621169153, 'mean_word_length': 1.975609756097561, 'pattern_counts': {'":': 0, '<': 19, '<?xml version=': 0, '>': 275, 'https://': 2, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 8, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '5683419', 'n_tokens_mistral': 16713, 'n_tokens_neox': 16045, 'n_words': 5939}