<commit_before># SPDX-License-Identifier: Apache-2.0

import sys

import numpy as np  # type: ignore
import numpy.typing as nptyping  # type: ignore
from onnx import TensorProto, MapProto, SequenceProto, OptionalProto
from onnx import mapping, helper
from onnx.external_data_helper import load_external_data_for_tensor, uses_external_data
from typing import Sequence, Any, Optional, List, Dict


def combine_pairs_to_complex(fa: Sequence[int]) -> Sequence[np.complex64]:
    return [complex(fa[i * 2], fa[i * 2 + 1]) for i in range(len(fa) // 2)]


def bfloat16_to_float32(data: np.ndarray, dims: nptyping._ShapeLike) -> np.ndarray:
    """Converts ndarray of bf16 (as uint32) to f32 (as uint32)."""
    shift = lambda x: x << 16  # noqa: E731
    return shift(data.astype(np.int32)).reshape(dims).view(np.float32)


def to_array(tensor: TensorProto, base_dir: str = "") -> np.ndarray:
    """Converts a tensor def object to a numpy array.
<commit_msg>Remove reference to numpy.typing (#4277)<commit_after># SPDX-License-Identifier: Apache-2.0

import sys
from typing import Any, Dict, List, Optional, Sequence, Union

import numpy as np  # type: ignore

from onnx import MapProto, OptionalProto, SequenceProto, TensorProto, mapping
from onnx.external_data_helper import load_external_data_for_tensor, uses_external_data


def combine_pairs_to_complex(fa: Sequence[int]) -> List[complex]:
    return [complex(fa[i * 2], fa[i * 2 + 1]) for i in range(len(fa) // 2)]


def bfloat16_to_float32(data: np.ndarray, dims: Union[int, Sequence[int]]) -> np.ndarray:
    """Converts ndarray of bf16 (as uint32) to f32 (as uint32)."""
    shift = lambda x: x << 16  # noqa: E731
    return shift(data.astype(np.int32)).reshape(dims).view(np.float32)


def to_array(tensor: TensorProto, base_dir: str = "") -> np.ndarray:
    """Converts a tensor def object to a numpy array.

{'fraction_non_alphanumeric': 0.12070815450643776, 'fraction_numerical': 0.032188841201716736, 'mean_word_length': 4.756172839506172, 'pattern_counts': {'":': 0, '<': 7, '<?xml version=': 0, '>': 9, 'https://': 0, 'lorem ipsum': 0, 'www.': 0, 'xml': 0}, 'pii_count': 0, 'substrings_counts': 0, 'word_list_counts': {'cursed_substrings.json': 0, 'profanity_word_list.json': 0, 'sexual_word_list.json': 0, 'zh_pornsignals.json': 0}}
{'dir': 'git-commits-cleaned', 'id': '6571788', 'n_tokens_mistral': 677, 'n_tokens_neox': 630, 'n_words': 190}