| 1 | import torch |
| 2 | |
| 3 | |
| 4 | class GELUApprox(torch.nn.Module): |
| 5 | def __init__(self, dim_in: int, dim_out: int) -> None: |
| 6 | super().__init__() |
| 7 | self.proj = torch.nn.Linear(dim_in, dim_out) |
| 8 | |
| 9 | def forward(self, x: torch.Tensor) -> torch.Tensor: |
| 10 | return torch.nn.functional.gelu(self.proj(x), approximate="tanh") |
| 11 |