# Path: flaxformer/types.py



from typing import Any, Callable
from flax import linen as nn
from flax.linen import partitioning
from jax import lax
from flaxformer.types import Array
import jax.numpy as jnp
import numpy as np
head effectively attends to things at different scales, irrespective of the
contents of keys and queries.

In the future, this class may be unified with classes which take into account
key and query contents, like the original relative position embeddings of Shaw
et al. and new proposals. However, this will rely on XLA to recover efficiency
for this class (especially when, as in the original T5, the same bias matrix
is shared for all layers).
"""




class RelativePositionBiases(nn.Module):
  """Adds T5-style relative positional embeddings to the attention logits.

  Attributes:
    num_buckets: Number of buckets to bucket distances between key and query
      positions into.
    max_distance: Maximum distance before everything is lumped into the last
      distance bucket.
    num_heads: Number of heads in the attention layer. Each head will get a
      different relative position weighting.
    dtype: Type of arrays through this module.
    embedding_init: initializer for relative embedding table.
  """
  num_buckets: int
  max_distance: int
  num_heads: int
  dtype: Any

# Path: flaxformer/types.py



# Path: flaxformer/components/relative_position_biases.py
from typing import Any, Callable
from flax import linen as nn
from flax.linen import partitioning
from jax import lax
from flaxformer.types import Array
import jax.numpy as jnp
import numpy as np

head effectively attends to things at different scales, irrespective of the
contents of keys and queries.

In the future, this class may be unified with classes which take into account
key and query contents, like the original relative position embeddings of Shaw
et al. and new proposals. However, this will rely on XLA to recover efficiency
for this class (especially when, as in the original T5, the same bias matrix
is shared for all layers).
"""




class RelativePositionBiases(nn.Module):
  """Adds T5-style relative positional embeddings to the attention logits.

  Attributes:
    num_buckets: Number of buckets to bucket distances between key and query
      positions into.
    max_distance: Maximum distance before everything is lumped into the last
      distance bucket.
    num_heads: Number of heads in the attention layer. Each head will get a
      different relative position weighting.
    dtype: Type of arrays through this module.
    embedding_init: initializer for relative embedding table.
  """
  num_buckets: int
  max_distance: int
  num_heads: int
  dtype: Any

  embedding_init: Callable[..., Array] = nn.linear.default_embed_init