import math import torch from torch import nn # adapted from https://pytorch.org/tutorials/beginner/transformer_tutorial.html class PositionEmbedding1D(nn.Module): def __init__(self, embedding_dim, dropout=0.1, max_len=128): super().__init__() # self.dropout = nn.Dropout(p=dropout) position = torch.arange(max_len).unsqueeze(1) div_term = torch.exp(torch.arange(0, embedding_dim, 2) * (-math.log(10000.0) / embedding_dim)) pe = torch.zeros(max_len, embedding_dim) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0) # .transpose(0, 1) self.register_buffer('pe', pe) def forward(self, x): # # x: Tensor, shape [batch_size, seq_len, embedding_dim] # x = x + self.pe[:, :x.size(1)] # return self.dropout(x) N, T, _ = x.size() return self.pe[:, :T].repeat(N, 1, 1) class LearnedPositionEmbedding1D(nn.Module): def __init__(self, embedding_dim, max_len=128): super().__init__() self.pe = nn.Parameter(torch.Tensor(1, max_len, embedding_dim)) self.reset_parameters() def reset_parameters(self): nn.init.xavier_normal_(self.pe) def forward(self, x): N, T, _ = x.size() return self.pe[:, :T].repeat(N, 1, 1) # https://huggingface.co/transformers/_modules/transformers/models/detr/modeling_detr.html class PositionEmbedding2D(nn.Module): def __init__(self, embedding_dim, temperature=10000, normalize=False, scale=None): super().__init__() assert embedding_dim % 2 == 0 self.half_embedding_dim = embedding_dim // 2 self.temperature = temperature self.normalize = normalize if scale is not None and normalize is False: raise ValueError("normalize should be True if scale is passed") if scale is None: scale = 2 * math.pi self.scale = scale def forward(self, pixel_values, pixel_mask): assert pixel_mask is not None, "No pixel mask provided" if pixel_mask.dim() == 4 and pixel_mask.size(1) == 1: pixel_mask = pixel_mask.squeeze(1) y_embed = pixel_mask.cumsum(1, dtype=torch.float32) x_embed = pixel_mask.cumsum(2, dtype=torch.float32) if self.normalize: y_embed = y_embed / (y_embed[:, -1:, :] + 1e-6) * self.scale x_embed = x_embed / (x_embed[:, :, -1:] + 1e-6) * self.scale dim_t = torch.arange(self.half_embedding_dim, dtype=torch.float32, device=pixel_values.device) dim_t = self.temperature ** (2 * torch.divide(dim_t, 2, rounding_mode='floor') / self.half_embedding_dim) pos_x = x_embed[:, :, :, None] / dim_t pos_y = y_embed[:, :, :, None] / dim_t pos_x = torch.stack(( pos_x[:, :, :, 0::2].sin(), pos_x[:, :, :, 1::2].cos()), dim=4).flatten(3) pos_y = torch.stack(( pos_y[:, :, :, 0::2].sin(), pos_y[:, :, :, 1::2].cos()), dim=4).flatten(3) pos = torch.cat((pos_y, pos_x), dim=3).permute(0, 3, 1, 2) return pos # https://huggingface.co/transformers/_modules/transformers/models/detr/modeling_detr.html class LearnedPositionEmbedding2D(nn.Module): def __init__(self, embedding_dim): super().__init__() assert embedding_dim % 2 == 0, 'embedding dimensionality must be even' self.rows_embeddings = nn.Embedding(50, embedding_dim//2) self.cols_embeddings = nn.Embedding(50, embedding_dim//2) def forward(self, pixel_values, pixel_mask=None): h, w = pixel_values.shape[-2:] i = torch.arange(w, device=pixel_values.device) j = torch.arange(h, device=pixel_values.device) x_emb = self.cols_embeddings(i) y_emb = self.rows_embeddings(j) pos = torch.cat([x_emb.unsqueeze(0).repeat(h, 1, 1), y_emb.unsqueeze(1).repeat(1, w, 1)], dim=-1) pos = pos.permute(2, 0, 1) pos = pos.unsqueeze(0) pos = pos.repeat(pixel_values.shape[0], 1, 1, 1) return pos class Box8PositionEmbedding2D(nn.Module): def __init__(self, embedding_dim, with_projection=True): super().__init__() self.proj = None if with_projection: self.proj = nn.Linear(8, embedding_dim) nn.init.xavier_normal_(self.proj.weight) nn.init.zeros_(self.proj.bias) def forward(self, fmap, fmap_mask=None): N, _, H, W = fmap.size() y1, x1 = torch.meshgrid( torch.arange(H, device=fmap.device, dtype=torch.float)/H, torch.arange(W, device=fmap.device, dtype=torch.float)/W ) y2, x2 = x1+1.0/W, y1+1.0/H ww, hh = x2-x1, y2-y1 # x1, y1 = 2*x1-1, 2*y1-1 # x2, y2 = 2*x2-1, 2*y2-1 xc, yc = x1+0.5/W, y1+0.5/H pos = torch.stack([x1, y1, x2, y2, xc, yc, ww, hh], dim=-1) if self.proj is not None: pos = self.proj(pos) pos = pos.permute(2, 0, 1) pos = pos.unsqueeze(0).repeat(N, 1, 1, 1) return pos def encode_boxes(self, boxes): x1, y1, x2, y2 = boxes.unbind(-1) ww, hh = x2-x1, y2-y1 xc, yc = x1+0.5*ww, y1+0.5*hh pos = torch.stack([x1, y1, x2, y2, xc, yc, ww, hh], dim=-1) if self.proj is not None: pos = self.proj(pos) return pos class RelativePositionEmbedding2D(nn.Module): def __init__(self, embedding_dim, spatial_bins=(16, 16), with_projection=True): super().__init__() assert isinstance(spatial_bins, (list, tuple)) and len(spatial_bins) == 2 self.spatial_bins = spatial_bins self.proj = None if with_projection: self.proj = nn.Linear(2*spatial_bins[0]*spatial_bins[1], embedding_dim) nn.init.xavier_normal_(self.proj.weight) nn.init.zeros_(self.proj.bias) def forward(self, fmap, fmap_mask=None): N, _, H, W = fmap.size() BH, BW = self.spatial_bins yc, xc = torch.meshgrid( 0.5/BH + torch.arange(BH, device=fmap.device, dtype=torch.float)/BH, 0.5/BW + torch.arange(BW, device=fmap.device, dtype=torch.float)/BW ) pos = torch.stack([xc, yc], dim=-1).view(-1, 1, 2) pos = (pos - pos.transpose(0, 1)).reshape(BH, BW, -1) # relative positions if self.proj is not None: pos = self.proj(pos) pos = pos.permute(2, 0, 1) pos = pos.unsqueeze(0) if H != BH or W != BW: pos = nn.functional.interpolate(pos, (H, W), mode='nearest') pos = pos.repeat(N, 1, 1, 1) return pos