| 1 | import logging |
| 2 | from dataclasses import dataclass, field, replace |
| 3 | |
| 4 | from safetensors import safe_open |
| 5 | |
| 6 | from ltx_core.components.guiders import MultiModalGuiderParams |
| 7 | from ltx_core.types import SpatioTemporalScaleFactors |
| 8 | |
| 9 | # ============================================================================= |
| 10 | # Diffusion Schedule |
| 11 | # ============================================================================= |
| 12 | |
| 13 | # Noise schedule for the distilled pipeline. These sigma values control noise |
| 14 | # levels at each denoising step and were tuned to match the distillation process. |
| 15 | DISTILLED_SIGMA_VALUES = [1.0, 0.99375, 0.9875, 0.98125, 0.975, 0.909375, 0.725, 0.421875, 0.0] |
| 16 | |
| 17 | # Reduced schedule for super-resolution stage 2 (subset of distilled values) |
| 18 | STAGE_2_DISTILLED_SIGMA_VALUES = [0.909375, 0.725, 0.421875, 0.0] |
| 19 | |
| 20 | |
| 21 | # ============================================================================= |
| 22 | # Pipeline Parameters |
| 23 | # ============================================================================= |
| 24 | |
| 25 | |
| 26 | @dataclass(frozen=True) |
| 27 | class PipelineParams: |
| 28 | seed: int = 10 |
| 29 | stage_1_height: int = 512 |
| 30 | stage_1_width: int = 768 |
| 31 | num_frames: int = 121 |
| 32 | frame_rate: float = 24.0 |
| 33 | num_inference_steps: int = 40 |
| 34 | video_guider_params: MultiModalGuiderParams = field( |
| 35 | default_factory=lambda: MultiModalGuiderParams( |
| 36 | cfg_scale=3.0, |
| 37 | stg_scale=1.0, |
| 38 | rescale_scale=0.7, |
| 39 | modality_scale=3.0, |
| 40 | skip_step=0, |
| 41 | stg_blocks=[29], |
| 42 | ) |
| 43 | ) |
| 44 | audio_guider_params: MultiModalGuiderParams = field( |
| 45 | default_factory=lambda: MultiModalGuiderParams( |
| 46 | cfg_scale=7.0, |
| 47 | stg_scale=1.0, |
| 48 | rescale_scale=0.7, |
| 49 | modality_scale=3.0, |
| 50 | skip_step=0, |
| 51 | stg_blocks=[29], |
| 52 | ) |
| 53 | ) |
| 54 | |
| 55 | @property |
| 56 | def stage_2_height(self) -> int: |
| 57 | return int(self.stage_1_height * 2) |
| 58 | |
| 59 | @property |
| 60 | def stage_2_width(self) -> int: |
| 61 | return int(self.stage_1_width * 2) |
| 62 | |
| 63 | |
| 64 | # Default params for LTX-2.0 non-distilled models. These can be overridden by detecting from checkpoint metadata. |
| 65 | LTX_2_PARAMS = PipelineParams() |
| 66 | |
| 67 | # Default params for LTX-2.3 non-distilled models. These override some of the LTX-2.0 defaults. |
| 68 | LTX_2_3_PARAMS = replace( |
| 69 | LTX_2_PARAMS, |
| 70 | num_inference_steps=30, |
| 71 | video_guider_params=replace(LTX_2_PARAMS.video_guider_params, stg_blocks=[28]), |
| 72 | audio_guider_params=replace(LTX_2_PARAMS.audio_guider_params, stg_blocks=[28]), |
| 73 | ) |
| 74 | LTX_2_3_HQ_PARAMS = PipelineParams( |
| 75 | num_inference_steps=15, |
| 76 | stage_1_height=1088 // 2, |
| 77 | stage_1_width=1920 // 2, |
| 78 | video_guider_params=MultiModalGuiderParams( |
| 79 | cfg_scale=3.0, |
| 80 | stg_scale=0.0, |
| 81 | rescale_scale=0.45, |
| 82 | modality_scale=3.0, |
| 83 | skip_step=0, |
| 84 | stg_blocks=[], |
| 85 | ), |
| 86 | audio_guider_params=MultiModalGuiderParams( |
| 87 | cfg_scale=7.0, |
| 88 | stg_scale=0.0, |
| 89 | rescale_scale=1.0, |
| 90 | modality_scale=3.0, |
| 91 | skip_step=0, |
| 92 | stg_blocks=[], |
| 93 | ), |
| 94 | ) |
| 95 | |
| 96 | DEFAULT_LORA_STRENGTH = 1.0 |
| 97 | DEFAULT_IMAGE_CRF = 33 |
| 98 | VIDEO_SCALE_FACTORS = SpatioTemporalScaleFactors.default() |
| 99 | VIDEO_LATENT_CHANNELS = 128 |
| 100 | |
| 101 | _LTX_2_3_MODEL_VERSION_PREFIX = "2.3" |
| 102 | |
| 103 | |
| 104 | def detect_params(checkpoint_path: str) -> PipelineParams: |
| 105 | """Detect pipeline params from checkpoint metadata. |
| 106 | Reads the ``model_version`` field from the safetensors config metadata. |
| 107 | Returns ``LTX_2_3_PARAMS`` when the version starts with "2.3", |
| 108 | otherwise falls back to ``LTX_2_PARAMS``. |
| 109 | """ |
| 110 | logger = logging.getLogger(__name__) |
| 111 | |
| 112 | try: |
| 113 | with safe_open(checkpoint_path, framework="pt") as f: |
| 114 | metadata = f.metadata() or {} |
| 115 | version = metadata.get("model_version", "") |
| 116 | except Exception: |
| 117 | logger.warning("Could not read checkpoint metadata from %s, using LTX-2 defaults", checkpoint_path) |
| 118 | return LTX_2_PARAMS |
| 119 | |
| 120 | if version.startswith(_LTX_2_3_MODEL_VERSION_PREFIX): |
| 121 | return LTX_2_3_PARAMS |
| 122 | |
| 123 | logger.info("Using LTX_2_PARAMS for checkpoint (version=%s)", version or "unknown") |
| 124 | return LTX_2_PARAMS |
| 125 | |
| 126 | |
| 127 | # ============================================================================= |
| 128 | # Prompts |
| 129 | # ============================================================================= |
| 130 | |
| 131 | DEFAULT_NEGATIVE_PROMPT = ( |
| 132 | "blurry, out of focus, overexposed, underexposed, low contrast, washed out colors, excessive noise, " |
| 133 | "grainy texture, poor lighting, flickering, motion blur, distorted proportions, unnatural skin tones, " |
| 134 | "deformed facial features, asymmetrical face, missing facial features, extra limbs, disfigured hands, " |
| 135 | "wrong hand count, artifacts around text, inconsistent perspective, camera shake, incorrect depth of " |
| 136 | "field, background too sharp, background clutter, distracting reflections, harsh shadows, inconsistent " |
| 137 | "lighting direction, color banding, cartoonish rendering, 3D CGI look, unrealistic materials, uncanny " |
| 138 | "valley effect, incorrect ethnicity, wrong gender, exaggerated expressions, wrong gaze direction, " |
| 139 | "mismatched lip sync, silent or muted audio, distorted voice, robotic voice, echo, background noise, " |
| 140 | "off-sync audio, incorrect dialogue, added dialogue, repetitive speech, jittery movement, awkward " |
| 141 | "pauses, incorrect timing, unnatural transitions, inconsistent framing, tilted camera, flat lighting, " |
| 142 | "inconsistent tone, cinematic oversaturation, stylized filters, or AI artifacts." |
| 143 | ) |
| 144 |