| 1 | from typing import List, Optional, Literal |
| 2 | import asyncio |
| 3 | from pydantic import BaseModel, Field |
| 4 | from tenacity import retry, stop_after_attempt |
| 5 | |
| 6 | from langchain.chat_models.base import BaseChatModel |
| 7 | from langchain_core.prompts import ChatPromptTemplate |
| 8 | from langchain_core.output_parsers import PydanticOutputParser |
| 9 | from interfaces import CharacterInScene, ShotDescription, ShotBriefDescription |
| 10 | |
| 11 | from utils.retry import after_func |
| 12 | |
| 13 | |
| 14 | |
| 15 | system_prompt_template_design_storyboard = \ |
| 16 | """ |
| 17 | [Role] |
| 18 | You are a professional storyboard artist with the following core skills: |
| 19 | - Script Analysis: Ability to quickly interpret a script's text, identifying the setting, character actions, dialogue, emotions, and narrative pacing. |
| 20 | - Visualization: Expertise in translating written descriptions into visual frames, including composition, lighting, and spatial arrangement. |
| 21 | - Storyboarding: Proficiency in cinematic language, such as shot types (e.g., close-up, medium shot, wide shot), camera angles (e.g., high angle, eye-level), camera movements (e.g., zoom, pan), and transitions. |
| 22 | - Narrative Continuity: Ability to ensure the storyboard sequence is logically smooth, highlights key plot points, and maintains emotional consistency. |
| 23 | - Technical Knowledge: Understanding of basic storyboard formats and industry standards, such as using numbered shots and concise descriptions. |
| 24 | |
| 25 | [Task] |
| 26 | Your task is to design a complete storyboard based on a user-provided script (which contains only one scene). The storyboard should be presented in text form, clearly displaying the visual elements and narrative flow of each shot to help the user visualize the scene. |
| 27 | |
| 28 | [Input] |
| 29 | The user will provide the following input. |
| 30 | - Script:A complete scene script containing dialogue, action descriptions, and scene settings. The script focuses on only one scene; there is no need to handle multiple scene transitions. The script input is enclosed within <SCRIPT> and </SCRIPT>. |
| 31 | - Characters List: A list describing basic information for each character, such as name, personality traits, appearance (if relevant). The character list is enclosed within <CHARACTERS> and </CHARACTERS>. |
| 32 | - User requirement: The user requirement (optional) is enclosed within <USER_REQUIREMENT> and </USER_REQUIREMENT>, which may include: |
| 33 | - Target audience (e.g., children, teenagers, adults). |
| 34 | - Storyboard style (e.g., realistic, cartoon, abstract). |
| 35 | - Desired number of shots (e.g., "not more than 10 shots"). |
| 36 | - Other specific instructions (e.g., emphasize the characters' actions). |
| 37 | |
| 38 | [Output] |
| 39 | {format_instructions} |
| 40 | |
| 41 | [Guidelines] |
| 42 | - Ensure all output values (except keys) match the language used in the script. |
| 43 | - Each shot must have a clear narrative purpose—such as establishing the setting, showing character relationships, or highlighting reactions. |
| 44 | - Use cinematic language deliberately: close-ups for emotion, wide shots for context, and varied angles to direct audience attention. |
| 45 | - When designing a new shot, first consider whether it can be filmed using an existing camera position. Introduce a new one only if the shot size, angle, and focus differ significantly. If the camera undergoes significant movement, it cannot be used thereafter. |
| 46 | - Keep character names in visual descriptions and speaker fields consistent with the character list. In visual descriptions, enclose names in angle brackets (e.g., <Alice>), but not in dialogue or speaker fields. |
| 47 | - When describing visual elements, it is necessary to indicate the position of the element within the frame. For example, Character A is on the left side of the frame, facing toward the right, with a table in front of him. The table is positioned slightly to the left of the center of the frame. Ensure that invisible elements are not included. For instance, do not describe someone behind a closed door if they cannot be seen. |
| 48 | - Avoid unsafe content (violence, discrimination, etc.) in visual descriptions. Use indirect methods like sound or suggestive imagery when needed, and substitute sensitive elements (e.g., ketchup for blood). |
| 49 | - Assign at most one dialogue line per character per shot. Each line of dialogue should correspond to a shot. |
| 50 | - Each shot requires an independent description without reference to each other. |
| 51 | - When the shot focuses on a character, describe which specific body part the focus is on. |
| 52 | - When describing a character, it is necessary to indicate the direction they are facing. |
| 53 | """ |
| 54 | |
| 55 | |
| 56 | human_prompt_template_design_storyboard = \ |
| 57 | """ |
| 58 | <SCRIPT> |
| 59 | {script_str} |
| 60 | </SCRIPT> |
| 61 | |
| 62 | <CHARACTERS> |
| 63 | {characters_str} |
| 64 | </CHARACTERS> |
| 65 | |
| 66 | <USER_REQUIREMENT> |
| 67 | {user_requirement_str} |
| 68 | </USER_REQUIREMENT> |
| 69 | """ |
| 70 | |
| 71 | |
| 72 | |
| 73 | system_prompt_template_decompose_visual_description = \ |
| 74 | """ |
| 75 | [Role] |
| 76 | You are a professional visual text analyst, proficient in cinematic language and shot narration. Your expertise lies in deconstructing a comprehensive shot description accurately into three core components: the static first frame, the static last frame, and the dynamic motion that connects them. |
| 77 | |
| 78 | [Task] |
| 79 | Your task is to dissect and rewrite a user-provided visual text description of a shot strictly and insightfully into three distinct parts: |
| 80 | - First Frame Description: Describe the static image at the very beginning of the shot. Focus on compositional elements, initial character postures, environmental layout, lighting, color, and other static visual aspects. |
| 81 | - Last Frame Description: Describe the static image at the very end of the shot. Similarly, focus on the static composition, but it must reflect the final state after changes caused by camera movement or internal element motion. |
| 82 | - Motion Description: Describe all movements that occur between the first frame and the last frame. This includes camera movement (e.g., static, push-in, pull-out, pan, track, follow, tilt, etc.) and movement of elements within the shot (e.g., character movement, object displacement, changes in lighting, etc.). This is the most dynamic part of the entire description. For the movement and changes of a character, you cannot directly use the character's name to refer to them. Instead, you need to refer to the character by their external features, especially noticeable ones like clothing characteristics. |
| 83 | |
| 84 | [Input] |
| 85 | You will receive a single visual text description of a shot that typically implicitly or explicitly contains information about the starting state, the motion process, and the ending state. |
| 86 | Additionally, you will receive a sequence of potential characters, each containing an identifier and a feature. |
| 87 | - The description is enclosed within <VISUAL_DESC> and </VISUAL_DESC>. |
| 88 | - The character list is enclosed within <CHARACTERS> and </CHARACTERS>. |
| 89 | |
| 90 | |
| 91 | [Output] |
| 92 | {format_instructions} |
| 93 | |
| 94 | [Guidelines] |
| 95 | - Ensure all output values (except keys) match the language used in the script. |
| 96 | - Ensure the first and last frame descriptions are pure "snapshots," containing no ongoing actions (e.g., "He is about to stand up" is unacceptable; it should be "He is sitting on the chair, leaning slightly forward"). |
| 97 | - In the motion description, you must clearly distinguish between camera movement and on-screen movement. Use professional cinematic terminology (e.g., dolly shot, pan, zoom, etc.) as precisely as possible to describe camera movement. |
| 98 | - In the motion description, you cannot directly use character names to refer to characters; instead, you should use the characters' visible characteristics to refer to them. For example, "Alice is walking" is unacceptable; it should be "Alice (short hair, wearing a green dress) is walking". |
| 99 | - The last frame description must be logically consistent with the first frame description and the motion description. All actions described in the motion section should be reflected in the static image of the last frame. |
| 100 | - If the input description is ambiguous about certain details, you may make reasonable inferences and additions based on the context to make all three sections complete and fluent. However, core elements must strictly adhere to the input text. |
| 101 | - Use accurate, concise, and professional descriptive language. Avoid overly literary rhetoric such as metaphors or emotional flourishes; focus on providing information that can be visualized. |
| 102 | - Similar to the input visual description, the first and last frame descriptions should include details such as shot type, angle, composition, etc. |
| 103 | - Below are the three types of variation within a shot (not between two shots): |
| 104 | (1) 'large' cases typically involve the exaggerated transition shots which means a significant change in the composition and focus, such as smoothly changing from a wide shot to a close-up. It is usually accompanied by significant camera movement (e.g., drone perspective shots across the city). |
| 105 | (2) 'medium' cases often involve the introduction of new characters and a character turns from the back to face the front (facing the camera). |
| 106 | (3) 'small' cases usually involve minor changes, such as expression changes, movement and pose changes of existing characters(e.g., walking, sitting down, standing up), moderate camera movements(e.g., pan, tilt, track). |
| 107 | - When describing a character, it is necessary to indicate the direction they are facing. |
| 108 | - The first shot must establish the overall scene environment, using the widest possible shot. |
| 109 | - Use as few camera positions as possible. |
| 110 | """ |
| 111 | |
| 112 | |
| 113 | human_prompt_template_decompose_visual_description = \ |
| 114 | """ |
| 115 | <VISUAL_DESC> |
| 116 | {visual_desc} |
| 117 | </VISUAL_DESC> |
| 118 | |
| 119 | <CHARACTERS> |
| 120 | {characters_str} |
| 121 | </CHARACTERS> |
| 122 | """ |
| 123 | |
| 124 | |
| 125 | class VisDescDecompositionResponse(BaseModel): |
| 126 | ff_desc: str = Field( |
| 127 | description="A detailed description of the first frame of the shot, capturing the initial visual elements and composition.", |
| 128 | # examples=[ |
| 129 | # "Medium shot of a supermarket aisle at eye level. Bob(a tall man wearing a blue shirt and jeans) is positioned on the right side of the frame, captured in profile and facing right, while Alice(a young woman with short hair, wearing a green dress) is on the left, shown pushing a shopping cart with her gaze lowered toward the ground. They are arranged in a front-to-back spatial relationship. Shelves line both sides of the frame, and cool-toned fluorescent lighting from above washes over the scene. The vibrant colors of product packaging contrast with the metallic gray of the shopping cart, all contained within a stable, horizontally balanced composition.", |
| 130 | # "Extreme long shot. Aerial view from hundreds of meters above the ground. The boundless golden desert resembles undulating frozen waves, occupying the vast majority of the frame. At the very center of the image, a tiny, solitary explorer appears only as a faint dark speck, dragging a long, lonely trail of footprints behind him, stretching all the way to the edge of the frame.", |
| 131 | # "Medium shot at eye level angle. Designer A(with a beard, wearing a white suit) leans forward passionately, speaking emphatically. Product Manager B(with a beard, wearing a white T-shirt) sits with crossed arms, looking skeptical. Between them, Development Engineer C(brown hair, wearing a blue T-shirt) appears anxious, glancing between the two. Project Manager D(curly hair, wearing a red T-shirt) prepares to mediate, focusing on a whiteboard. Bright overhead lighting highlights their expressions, with a blurred whiteboard and glass wall in the background.", |
| 132 | # "A low-angle close-up shot captures the figure from below, framing him from the chest up. His face appears resolute and commanding, his eyes piercing as he speaks passionately. Flecks of saliva are visible, emphasizing his intensity. The overcast sky breaks with occasional light, casting him as a heroic, almost monumental figure against the gloom.", |
| 133 | # "An extremely close-up of an old, motionless pocket watch. Soft light highlights scratches on its brass case and the enamel dial with Roman numerals. The second hand remains fixed at 'VIII', casting a sharp shadow. A wrinkled finger gently touches the glass surface, evoking a tangible sense of stillness and time.", |
| 134 | # "An over-the-shoulder shot at eye level, positioned behind Character A(red hair, wearing a white T-shirt). The foreground, including A's shoulder and head, is softly blurred, directing focus onto Character B(with a beard, wearing a white T-shirt)'s face. B's subtle reactions—shifting from surprise to confusion, then to a glimmer of understanding—are clearly visible. The café background is gently blurred with warm lighting.", |
| 135 | # ] |
| 136 | ) |
| 137 | ff_vis_char_idxs: List[int] = Field( |
| 138 | description="A list of indices of characters that are visible in the first frame of the shot, corresponding to the character list provided in the input.", |
| 139 | examples=[[0], [1], [0, 1], []] |
| 140 | ) |
| 141 | lf_desc: str = Field( |
| 142 | description="A detailed description of the last frame of the shot, capturing the concluding visual elements and composition.", |
| 143 | ) |
| 144 | lf_vis_char_idxs: List[int] = Field( |
| 145 | description="A list of indices of characters that are visible in the last frame of the shot, corresponding to the character list provided in the input.", |
| 146 | examples=[[0], [1], [0, 1], []] |
| 147 | ) |
| 148 | motion_desc: str = Field( |
| 149 | description="The motion description of the shot. Describe the dynamic visual changes within the shot (camera movement and the movement of elements within the frame)", |
| 150 | examples=[ |
| 151 | "Static camera. Alice (short hair, wearing a green dress) is walking towards the camera.", |
| 152 | "Dolly in from meidum shot to close-up. Bob (with a beard, wearing a white T-shirt) smiles to the camera.", |
| 153 | ] |
| 154 | ) |
| 155 | variation_type: Literal["large", "medium", "small"] = Field( |
| 156 | description="Indicates the degree of change between the first frame and the last frame.", |
| 157 | ) |
| 158 | variation_reason: str = Field( |
| 159 | description="The reason for the variation type of the shot.", |
| 160 | examples=[ |
| 161 | "This is a smooth transition shot from the sky to the ground. The content of the shot has changed significantly, so the variation type is large.", |
| 162 | "Compared to the first frame, a new character appears in the last frame, and there are no significant changes in the composition. So the variation type is medium.", |
| 163 | "Compared to the first frame, there are only minor changes in the composition. So the variation type is small.", |
| 164 | "This shot only shows Alice speaking and the changes in her facial expressions, thus the variation type is small.", |
| 165 | ], |
| 166 | ) |
| 167 | |
| 168 | |
| 169 | |
| 170 | class StoryboardArtist: |
| 171 | def __init__( |
| 172 | self, |
| 173 | chat_model: BaseChatModel, |
| 174 | ): |
| 175 | self.chat_model = chat_model |
| 176 | |
| 177 | |
| 178 | @retry(stop=stop_after_attempt(3), after=after_func) |
| 179 | async def design_storyboard( |
| 180 | self, |
| 181 | script: str, |
| 182 | characters: List[CharacterInScene], |
| 183 | user_requirement: Optional[str] = None, |
| 184 | retry_timeout: int = 150, |
| 185 | ) -> List[ShotBriefDescription]: |
| 186 | |
| 187 | class StoryboardResponse(BaseModel): |
| 188 | storyboard: List[ShotBriefDescription] = Field( |
| 189 | description="A complete storyboard of the scene, including the visual and audio description of each shot.", |
| 190 | ) |
| 191 | |
| 192 | script_str = script.strip() |
| 193 | characters_str = "\n".join([f"Character {index}: {char}" for index, char in enumerate(characters)]) |
| 194 | user_requirement_str = user_requirement.strip() if user_requirement else "" |
| 195 | |
| 196 | parser = PydanticOutputParser(pydantic_object=StoryboardResponse) |
| 197 | messages = [ |
| 198 | ('system', system_prompt_template_design_storyboard.format(format_instructions=parser.get_format_instructions())), |
| 199 | ('human', human_prompt_template_design_storyboard.format(script_str=script_str, characters_str=characters_str, user_requirement_str=user_requirement_str)), |
| 200 | ] |
| 201 | chain = self.chat_model | parser |
| 202 | response: StoryboardResponse = await asyncio.wait_for( |
| 203 | chain.ainvoke(messages), |
| 204 | timeout=retry_timeout, |
| 205 | ) |
| 206 | storyboard = response.storyboard |
| 207 | |
| 208 | return storyboard |
| 209 | |
| 210 | |
| 211 | |
| 212 | |
| 213 | @retry(stop=stop_after_attempt(3), after=after_func) |
| 214 | async def decompose_visual_description( |
| 215 | self, |
| 216 | shot_brief_desc: ShotBriefDescription, |
| 217 | characters: List[CharacterInScene], |
| 218 | retry_timeout: int = 150, |
| 219 | ) -> ShotDescription: |
| 220 | parser = PydanticOutputParser(pydantic_object=VisDescDecompositionResponse) |
| 221 | prompt_template = ChatPromptTemplate.from_messages( |
| 222 | [ |
| 223 | ('system', system_prompt_template_decompose_visual_description), |
| 224 | ('human', human_prompt_template_decompose_visual_description), |
| 225 | ] |
| 226 | ) |
| 227 | chain = prompt_template | self.chat_model | parser |
| 228 | |
| 229 | visual_desc = shot_brief_desc.visual_desc.strip() |
| 230 | |
| 231 | characters_str = "\n".join([f"{char.identifier_in_scene}: (static) {char.static_features}; (dynamic) {char.dynamic_features}" for char in characters]) |
| 232 | |
| 233 | decomposition: VisDescDecompositionResponse = await asyncio.wait_for( |
| 234 | chain.ainvoke( |
| 235 | input={ |
| 236 | "format_instructions": parser.get_format_instructions(), |
| 237 | "visual_desc": visual_desc, |
| 238 | "characters_str": characters_str, |
| 239 | }, |
| 240 | ), |
| 241 | timeout=retry_timeout, |
| 242 | ) |
| 243 | |
| 244 | validate_char_idxs(decomposition.ff_vis_char_idxs, len(characters), "ff_vis_char_idxs") |
| 245 | validate_char_idxs(decomposition.lf_vis_char_idxs, len(characters), "lf_vis_char_idxs") |
| 246 | |
| 247 | return ShotDescription( |
| 248 | idx=shot_brief_desc.idx, |
| 249 | is_last=shot_brief_desc.is_last, |
| 250 | cam_idx=shot_brief_desc.cam_idx, |
| 251 | visual_desc=shot_brief_desc.visual_desc, |
| 252 | variation_type=decomposition.variation_type, |
| 253 | variation_reason=decomposition.variation_reason, |
| 254 | ff_desc=decomposition.ff_desc, |
| 255 | ff_vis_char_idxs=decomposition.ff_vis_char_idxs, |
| 256 | lf_desc=decomposition.lf_desc, |
| 257 | lf_vis_char_idxs=decomposition.lf_vis_char_idxs, |
| 258 | motion_desc=decomposition.motion_desc, |
| 259 | audio_desc=shot_brief_desc.audio_desc, |
| 260 | ) |
| 261 | |
| 262 | |
| 263 | def validate_char_idxs(idxs, num_characters, field_name): |
| 264 | """Reject LLM-emitted character indices outside [0, num_characters). |
| 265 | |
| 266 | Negative values would silently select the wrong character via Python |
| 267 | indexing; out-of-range values would crash deep inside the render gather. |
| 268 | Raising here lets the @retry on decompose_visual_description re-ask. |
| 269 | """ |
| 270 | invalid = [idx for idx in idxs if idx < 0 or idx >= num_characters] |
| 271 | if invalid: |
| 272 | raise ValueError( |
| 273 | f"{field_name} contains invalid character indices {invalid}; " |
| 274 | f"valid range is 0..{num_characters - 1}" |
| 275 | ) |
| 276 |