| 1 | from typing import Protocol, Tuple |
| 2 | |
| 3 | import torch |
| 4 | |
| 5 | from ltx_core.types import AudioLatentShape, VideoLatentShape |
| 6 | |
| 7 | |
| 8 | class Patchifier(Protocol): |
| 9 | """ |
| 10 | Protocol for patchifiers that convert latent tensors into patches and assemble them back. |
| 11 | """ |
| 12 | |
| 13 | def patchify( |
| 14 | self, |
| 15 | latents: torch.Tensor, |
| 16 | ) -> torch.Tensor: |
| 17 | ... |
| 18 | """ |
| 19 | Convert latent tensors into flattened patch tokens. |
| 20 | Args: |
| 21 | latents: Latent tensor to patchify. |
| 22 | Returns: |
| 23 | Flattened patch tokens tensor. |
| 24 | """ |
| 25 | |
| 26 | def unpatchify( |
| 27 | self, |
| 28 | latents: torch.Tensor, |
| 29 | output_shape: AudioLatentShape | VideoLatentShape, |
| 30 | ) -> torch.Tensor: |
| 31 | """ |
| 32 | Converts latent tensors between spatio-temporal formats and flattened sequence representations. |
| 33 | Args: |
| 34 | latents: Patch tokens that must be rearranged back into the latent grid constructed by `patchify`. |
| 35 | output_shape: Shape of the output tensor. Note that output_shape is either AudioLatentShape or |
| 36 | VideoLatentShape. |
| 37 | Returns: |
| 38 | Dense latent tensor restored from the flattened representation. |
| 39 | """ |
| 40 | |
| 41 | @property |
| 42 | def patch_size(self) -> Tuple[int, int, int]: |
| 43 | ... |
| 44 | """ |
| 45 | Returns the patch size as a tuple of (temporal, height, width) dimensions |
| 46 | """ |
| 47 | |
| 48 | def get_patch_grid_bounds( |
| 49 | self, |
| 50 | output_shape: AudioLatentShape | VideoLatentShape, |
| 51 | device: torch.device | None = None, |
| 52 | ) -> torch.Tensor: |
| 53 | ... |
| 54 | """ |
| 55 | Compute metadata describing where each latent patch resides within the |
| 56 | grid specified by `output_shape`. |
| 57 | Args: |
| 58 | output_shape: Target grid layout for the patches. |
| 59 | device: Target device for the returned tensor. |
| 60 | Returns: |
| 61 | Tensor containing patch coordinate metadata such as spatial or temporal intervals. |
| 62 | """ |
| 63 | |
| 64 | |
| 65 | class SchedulerProtocol(Protocol): |
| 66 | """ |
| 67 | Protocol for schedulers that provide a sigmas schedule tensor for a |
| 68 | given number of steps. Device is cpu. |
| 69 | """ |
| 70 | |
| 71 | def execute(self, steps: int, **kwargs) -> torch.FloatTensor: ... |
| 72 | |
| 73 | |
| 74 | class GuiderProtocol(Protocol): |
| 75 | """ |
| 76 | Protocol for guiders that compute a delta tensor given conditioning inputs. |
| 77 | The returned delta should be added to the conditional output (cond), enabling |
| 78 | multiple guiders to be chained together by accumulating their deltas. |
| 79 | """ |
| 80 | |
| 81 | scale: float |
| 82 | |
| 83 | def delta(self, cond: torch.Tensor, uncond: torch.Tensor) -> torch.Tensor: ... |
| 84 | |
| 85 | def enabled(self) -> bool: |
| 86 | """ |
| 87 | Returns whether the corresponding perturbation is enabled. E.g. for CFG, this should return False if the scale |
| 88 | is 1.0. |
| 89 | """ |
| 90 | ... |
| 91 | |
| 92 | |
| 93 | class DiffusionStepProtocol(Protocol): |
| 94 | """ |
| 95 | Protocol for diffusion steps that provide a next sample tensor for a given current sample tensor, |
| 96 | current denoised sample tensor, and sigmas tensor. |
| 97 | """ |
| 98 | |
| 99 | def step( |
| 100 | self, sample: torch.Tensor, denoised_sample: torch.Tensor, sigmas: torch.Tensor, step_index: int, **kwargs |
| 101 | ) -> torch.Tensor: ... |
| 102 |