# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py



from typing import Union
from jax import numpy as jnp
from flaxformer.types import Array
from flaxformer.types import DType
from flaxformer.types import Initializer
from flaxformer.types import PRNGKey
from flaxformer.types import Shape
import jax
import numpy as np
      raise ValueError(
          f'Expected a 2D shape (max_len, features), but got {shape}.')
    max_len, features = shape
    pe = np.zeros((max_len, features), dtype=dtype)
    position = np.arange(0, max_len)[:, np.newaxis]
    scale_factor = -np.log(max_scale / min_scale) / (features // 2 - 1)
    div_term = min_scale * np.exp(np.arange(0, features // 2) * scale_factor)
    pe[:, :features // 2] = np.sin(position * div_term)
    pe[:, features // 2:2 * (features // 2)] = np.cos(position * div_term)
    return jnp.array(pe)

  return init


def truncated_normal(mean: Union[float, Array] = 0.0,
                     stddev: Union[float, Array] = 0.05,
                     dtype: DType = jnp.float32) -> Initializer:
  """Returns an initialization function "truncated normal".

  This is the initialization that is used in the original BERT implementation.

  Args:
    mean: The mean of the random values to generate.
    stddev: The standard deviation of the random values to generate.
    dtype: dtype of the initialized values.

  Returns:
    The truncated normal initializer.
  """


# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py

# 
# Path: flaxformer/types.py



# Path: flaxformer/components/initializers.py
from typing import Union
from jax import numpy as jnp
from flaxformer.types import Array
from flaxformer.types import DType
from flaxformer.types import Initializer
from flaxformer.types import PRNGKey
from flaxformer.types import Shape
import jax
import numpy as np

      raise ValueError(
          f'Expected a 2D shape (max_len, features), but got {shape}.')
    max_len, features = shape
    pe = np.zeros((max_len, features), dtype=dtype)
    position = np.arange(0, max_len)[:, np.newaxis]
    scale_factor = -np.log(max_scale / min_scale) / (features // 2 - 1)
    div_term = min_scale * np.exp(np.arange(0, features // 2) * scale_factor)
    pe[:, :features // 2] = np.sin(position * div_term)
    pe[:, features // 2:2 * (features // 2)] = np.cos(position * div_term)
    return jnp.array(pe)

  return init


def truncated_normal(mean: Union[float, Array] = 0.0,
                     stddev: Union[float, Array] = 0.05,
                     dtype: DType = jnp.float32) -> Initializer:
  """Returns an initialization function "truncated normal".

  This is the initialization that is used in the original BERT implementation.

  Args:
    mean: The mean of the random values to generate.
    stddev: The standard deviation of the random values to generate.
    dtype: dtype of the initialized values.

  Returns:
    The truncated normal initializer.
  """


  def init(key: PRNGKey, shape: Shape, dtype: DType = dtype) -> Array: