返回 AiToEarn
image-edit.mcp.ts
根目录 / project / aitoearn-backend / apps / aitoearn-ai / src / core / agent / mcp / image-edit.mcp.ts
1 import { createSdkMcpServer, McpSdkServerConfigWithInstance } from '@anthropic-ai/claude-agent-sdk'
2 import { Injectable, Logger } from '@nestjs/common'
3 import { AssetsService } from '@yikart/assets'
4 import { UserType } from '@yikart/common'
5 import { AssetType } from '@yikart/mongodb'
6 import axios from 'axios'
7 import sharp, { Blend, FitEnum, Gravity } from 'sharp'
8 import { z } from 'zod'
9 import { AiAvailabilityService } from '../../ai-availability'
10 import { McpServerName } from '../agent.constants'
11 import { successResult, wrapTool } from './mcp.utils'
12
13 // ==================== Zod Schemas ====================
14
15 const imageInputSchema = z.string().describe('Image URL (http/https)')
16
17 const gravitySchema = z.enum([
18 'northwest',
19 'north',
20 'northeast',
21 'west',
22 'center',
23 'east',
24 'southwest',
25 'south',
26 'southeast',
27 ])
28
29 const fitSchema = z.enum(['cover', 'contain', 'fill', 'inside', 'outside'])
30
31 const blendSchema = z.enum([
32 'over',
33 'multiply',
34 'screen',
35 'overlay',
36 'darken',
37 'lighten',
38 'hard-light',
39 'soft-light',
40 'difference',
41 'exclusion',
42 ])
43
44 const layerResizeSchema = z.object({
45 width: z.number().int().positive().optional().describe('Target width in pixels'),
46 height: z.number().int().positive().optional().describe('Target height in pixels'),
47 scale: z.number().positive().optional().describe('Scale factor (e.g., 0.5 = half size)'),
48 fit: fitSchema.optional().default('contain').describe('Resize fit mode'),
49 })
50
51 const compositeLayerSchema = z.object({
52 input: imageInputSchema.describe('Image URL to overlay'),
53 resize: layerResizeSchema.optional().describe('Resize the layer before compositing'),
54 gravity: gravitySchema.optional().describe('Position anchor. Use ALONE without top/left.'),
55 top: z.number().int().optional().describe('Absolute Y from top edge. Use with left, WITHOUT gravity.'),
56 left: z.number().int().optional().describe('Absolute X from left edge. Use with top, WITHOUT gravity.'),
57 blend: blendSchema.optional().describe('Blend mode'),
58 opacity: z.number().min(0).max(1).optional().describe('Opacity 0-1'),
59 })
60
61 const compositeImagesSchema = z.object({
62 base: imageInputSchema.describe('Base image URL (e.g., poster)'),
63 layers: z.array(compositeLayerSchema).min(1).describe('Images to overlay'),
64 })
65
66 const resizeImageSchema = z.object({
67 input: imageInputSchema,
68 width: z.number().int().positive().optional().describe('Target width'),
69 height: z.number().int().positive().optional().describe('Target height'),
70 fit: fitSchema.optional().default('cover').describe('Resize fit mode'),
71 background: z.string().optional().describe('Background color (e.g., #ffffff)'),
72 extract: z.object({
73 left: z.number().int().min(0),
74 top: z.number().int().min(0),
75 width: z.number().int().positive(),
76 height: z.number().int().positive(),
77 }).optional().describe('Crop region'),
78 })
79
80 const transformImageSchema = z.object({
81 input: imageInputSchema,
82 rotate: z.number().optional().describe('Rotation angle in degrees'),
83 flip: z.boolean().optional().describe('Vertical flip'),
84 flop: z.boolean().optional().describe('Horizontal flip'),
85 })
86
87 const adjustImageSchema = z.object({
88 input: imageInputSchema,
89 modulate: z.object({
90 brightness: z.number().min(0).optional().describe('Brightness multiplier (1 = no change)'),
91 saturation: z.number().min(0).optional().describe('Saturation multiplier (1 = no change)'),
92 hue: z.number().optional().describe('Hue rotation in degrees'),
93 }).optional(),
94 sharpen: z.object({
95 sigma: z.number().min(0.01).max(10).describe('Sharpening sigma'),
96 }).optional(),
97 blur: z.number().min(0.3).max(100).optional().describe('Blur sigma'),
98 greyscale: z.boolean().optional().describe('Convert to greyscale'),
99 negate: z.boolean().optional().describe('Invert colors'),
100 })
101
102 const getImageMetadataSchema = z.object({
103 input: imageInputSchema,
104 })
105
106 // ==================== Tool Names ====================
107
108 export enum ImageEditToolName {
109 CompositeImages = 'compositeImages',
110 ResizeImage = 'resizeImage',
111 TransformImage = 'transformImage',
112 AdjustImage = 'adjustImage',
113 GetImageMetadata = 'getImageMetadata',
114 }
115
116 // ==================== Helper Functions ====================
117
118 async function loadImageFromUrl(url: string): Promise<Buffer> {
119 const response = await axios.get(url, {
120 responseType: 'arraybuffer',
121 timeout: 60000,
122 maxContentLength: 50 * 1024 * 1024, // 50MB limit
123 })
124 return Buffer.from(response.data)
125 }
126
127 function parseColor(color: string): { r: number, g: number, b: number, alpha: number } {
128 const hex = color.replace('#', '')
129 if (hex.length === 6) {
130 return {
131 r: Number.parseInt(hex.slice(0, 2), 16),
132 g: Number.parseInt(hex.slice(2, 4), 16),
133 b: Number.parseInt(hex.slice(4, 6), 16),
134 alpha: 1,
135 }
136 }
137 if (hex.length === 8) {
138 return {
139 r: Number.parseInt(hex.slice(0, 2), 16),
140 g: Number.parseInt(hex.slice(2, 4), 16),
141 b: Number.parseInt(hex.slice(4, 6), 16),
142 alpha: Number.parseInt(hex.slice(6, 8), 16) / 255,
143 }
144 }
145 return { r: 255, g: 255, b: 255, alpha: 1 }
146 }
147
148 async function resizeLayer(
149 layerImage: sharp.Sharp,
150 resize: { width?: number, height?: number, scale?: number, fit?: string } | undefined,
151 originalWidth: number,
152 originalHeight: number,
153 ): Promise<{ image: sharp.Sharp, width: number, height: number }> {
154 if (!resize) {
155 return { image: layerImage, width: originalWidth, height: originalHeight }
156 }
157
158 let targetWidth: number | undefined
159 let targetHeight: number | undefined
160
161 if (resize.scale) {
162 targetWidth = Math.round(originalWidth * resize.scale)
163 targetHeight = Math.round(originalHeight * resize.scale)
164 }
165 else {
166 targetWidth = resize.width
167 targetHeight = resize.height
168 }
169
170 if (!targetWidth && !targetHeight) {
171 return { image: layerImage, width: originalWidth, height: originalHeight }
172 }
173
174 const resized = layerImage.resize({
175 width: targetWidth,
176 height: targetHeight,
177 fit: (resize.fit || 'contain') as keyof FitEnum,
178 background: { r: 0, g: 0, b: 0, alpha: 0 },
179 })
180
181 const metadata = await resized.metadata()
182 return { image: resized, width: metadata.width!, height: metadata.height! }
183 }
184
185 // ==================== MCP Service ====================
186
187 @Injectable()
188 export class ImageEditMcp {
189 private readonly logger = new Logger(ImageEditMcp.name)
190
191 constructor(
192 private readonly assetsService: AssetsService,
193 private readonly aiAvailability: AiAvailabilityService,
194 ) {}
195
196 private createCompositeImagesTool(userId: string, _userType: UserType) {
197 return wrapTool(
198 this.logger,
199 ImageEditToolName.CompositeImages,
200 `Composite multiple images onto a base image. Perfect for adding QR codes, logos, or watermarks to posters.
201
202 **Parameters:**
203 - base: Base image URL (e.g., poster background)
204 - layers: Array of overlay images with positioning
205 - input: Overlay image URL
206 - resize: Optional layer scaling
207 - width/height: Target dimensions in pixels
208 - scale: Scale factor (e.g., 0.5 = half size)
209 - fit: Resize mode (contain, cover, fill, inside, outside)
210 - gravity: Position anchor (northwest, north, northeast, west, center, east, southwest, south, southeast)
211 - top/left: Absolute pixel coordinates from top-left corner
212 - blend: Blend mode (over, multiply, screen, overlay, etc.)
213 - opacity: Transparency 0-1
214
215 **IMPORTANT - Two positioning modes (mutually exclusive):**
216
217 Mode A - gravity only (anchor positioning):
218 { "gravity": "southeast" } → places layer at bottom-right corner
219
220 Mode B - top/left only (absolute coordinates):
221 { "top": 100, "left": 200 } → places layer at exact pixel position
222
223 Do NOT mix gravity with top/left - if both provided, gravity is ignored by Sharp.
224
225 **Example - QR code at bottom-right corner:**
226 {
227 "base": "https://example.com/poster.jpg",
228 "layers": [{
229 "input": "https://example.com/qrcode.png",
230 "resize": { "width": 150, "height": 150 },
231 "gravity": "southeast"
232 }]
233 }
234
235 **Example - Logo at absolute position:**
236 {
237 "base": "https://example.com/poster.jpg",
238 "layers": [{
239 "input": "https://example.com/logo.png",
240 "resize": { "scale": 0.5 },
241 "top": 50,
242 "left": 50
243 }]
244 }`,
245 compositeImagesSchema.shape,
246 async ({ base, layers }) => {
247 this.logger.debug({ base, layerCount: layers.length }, '[compositeImages] Starting')
248
249 const baseBuffer = await loadImageFromUrl(base)
250 let image = sharp(baseBuffer)
251
252 const compositeInputs: sharp.OverlayOptions[] = []
253
254 for (const layer of layers) {
255 const layerBuffer = await loadImageFromUrl(layer.input)
256 let layerImage = sharp(layerBuffer)
257
258 // 1. 处理图层缩放
259 const layerMetadata = await sharp(layerBuffer).metadata()
260 const resized = await resizeLayer(
261 layerImage,
262 layer.resize,
263 layerMetadata.width!,
264 layerMetadata.height!,
265 )
266 layerImage = resized.image
267
268 // 2. 处理透明度
269 if (layer.opacity !== undefined && layer.opacity < 1) {
270 const { data, info } = await layerImage.ensureAlpha().raw().toBuffer({ resolveWithObject: true })
271 const pixels = new Uint8Array(data)
272 for (let i = 3; i < pixels.length; i += 4) {
273 pixels[i] = Math.round(pixels[i] * layer.opacity)
274 }
275 layerImage = sharp(Buffer.from(pixels), {
276 raw: { width: info.width, height: info.height, channels: 4 },
277 })
278 }
279
280 const processedLayerBuffer = await layerImage.toBuffer()
281
282 // 3. 直接传递定位参数给 Sharp
283 compositeInputs.push({
284 input: processedLayerBuffer,
285 gravity: layer.gravity as Gravity,
286 top: layer.top,
287 left: layer.left,
288 blend: (layer.blend || 'over') as Blend,
289 })
290 }
291
292 image = image.composite(compositeInputs)
293
294 const outputBuffer = await image.png({ compressionLevel: 9 }).toBuffer()
295
296 const result = await this.assetsService.uploadFromBuffer(userId, outputBuffer, {
297 type: AssetType.ImageEdit,
298 mimeType: 'image/png',
299 filename: 'composite.png',
300 }, 'image-edit')
301
302 this.logger.debug({ url: result.url }, '[compositeImages] Completed')
303 return {
304 content: [
305 {
306 type: 'resource_link',
307 uri: result.url,
308 name: `Image`,
309 },
310 { type: 'text', text: `Image composited successfully. URL: ${result.url}` },
311 ],
312 }
313 },
314 this.aiAvailability,
315 )
316 }
317
318 private createResizeImageTool(userId: string, _userType: UserType) {
319 return wrapTool(
320 this.logger,
321 ImageEditToolName.ResizeImage,
322 `Resize, crop, or extend an image.
323
324 **Parameters:**
325 - input: Image URL
326 - width/height: Target dimensions (at least one required for resize)
327 - fit: Resize mode
328 - cover: Crop to fill (default)
329 - contain: Fit within bounds, may add padding
330 - fill: Stretch to exact size
331 - inside: Fit within bounds, no upscaling
332 - outside: Cover bounds, may exceed
333 - background: Padding color for contain mode (e.g., #ffffff)
334 - extract: Crop region { left, top, width, height }
335
336 **Example - Resize to 800x600:**
337 { "input": "https://...", "width": 800, "height": 600, "fit": "cover" }
338
339 **Example - Crop center region:**
340 { "input": "https://...", "extract": { "left": 100, "top": 100, "width": 500, "height": 500 } }`,
341 resizeImageSchema.shape,
342 async ({ input, width, height, fit, background, extract }) => {
343 this.logger.debug({ input, width, height, fit, extract }, '[resizeImage] Starting')
344
345 const buffer = await loadImageFromUrl(input)
346 let image = sharp(buffer)
347
348 if (extract) {
349 image = image.extract(extract)
350 }
351
352 if (width || height) {
353 const resizeOptions: sharp.ResizeOptions = {
354 width,
355 height,
356 fit: fit as keyof FitEnum,
357 }
358 if (background) {
359 resizeOptions.background = parseColor(background)
360 }
361 image = image.resize(resizeOptions)
362 }
363
364 const outputBuffer = await image.png({ compressionLevel: 9 }).toBuffer()
365
366 const result = await this.assetsService.uploadFromBuffer(userId, outputBuffer, {
367 type: AssetType.ImageEdit,
368 mimeType: 'image/png',
369 filename: 'resized.png',
370 }, 'image-edit')
371
372 this.logger.debug({ url: result.url }, '[resizeImage] Completed')
373 return successResult(`Image resized successfully. URL: ${result.url}`)
374 },
375 this.aiAvailability,
376 )
377 }
378
379 private createTransformImageTool(userId: string, _userType: UserType) {
380 return wrapTool(
381 this.logger,
382 ImageEditToolName.TransformImage,
383 `Apply geometric transformations to an image.
384
385 **Parameters:**
386 - input: Image URL
387 - rotate: Rotation angle in degrees (positive = clockwise)
388 - flip: Vertical flip (mirror top-bottom)
389 - flop: Horizontal flip (mirror left-right)
390
391 **Example - Rotate 90 degrees:**
392 { "input": "https://...", "rotate": 90 }
393
394 **Example - Horizontal flip:**
395 { "input": "https://...", "flop": true }`,
396 transformImageSchema.shape,
397 async ({ input, rotate, flip, flop }) => {
398 this.logger.debug({ input, rotate, flip, flop }, '[transformImage] Starting')
399
400 const buffer = await loadImageFromUrl(input)
401 let image = sharp(buffer)
402
403 if (rotate !== undefined) {
404 image = image.rotate(rotate)
405 }
406 if (flip) {
407 image = image.flip()
408 }
409 if (flop) {
410 image = image.flop()
411 }
412
413 const outputBuffer = await image.png({ compressionLevel: 9 }).toBuffer()
414
415 const result = await this.assetsService.uploadFromBuffer(userId, outputBuffer, {
416 type: AssetType.ImageEdit,
417 mimeType: 'image/png',
418 filename: 'transformed.png',
419 }, 'image-edit')
420
421 this.logger.debug({ url: result.url }, '[transformImage] Completed')
422 return {
423 content: [
424 {
425 type: 'resource_link',
426 uri: result.url,
427 name: `Image`,
428 },
429 { type: 'text', text: `Image transformed successfully. URL: ${result.url}` },
430 ],
431 }
432 },
433 this.aiAvailability,
434 )
435 }
436
437 private createAdjustImageTool(userId: string, _userType: UserType) {
438 return wrapTool(
439 this.logger,
440 ImageEditToolName.AdjustImage,
441 `Adjust image appearance: brightness, contrast, saturation, sharpness, blur.
442
443 **Parameters:**
444 - input: Image URL
445 - modulate: Color adjustments
446 - brightness: Multiplier (1 = no change, 0.5 = darker, 2 = brighter)
447 - saturation: Multiplier (1 = no change, 0 = greyscale, 2 = vivid)
448 - hue: Rotation in degrees
449 - sharpen: { sigma: 0.01-10 } - higher = sharper
450 - blur: Blur sigma 0.3-100 - higher = more blur
451 - greyscale: Convert to black & white
452 - negate: Invert colors
453
454 **Example - Increase brightness and saturation:**
455 { "input": "https://...", "modulate": { "brightness": 1.2, "saturation": 1.3 } }
456
457 **Example - Apply blur:**
458 { "input": "https://...", "blur": 5 }`,
459 adjustImageSchema.shape,
460 async ({ input, modulate, sharpen, blur, greyscale, negate }) => {
461 this.logger.debug({ input, modulate, sharpen, blur, greyscale, negate }, '[adjustImage] Starting')
462
463 const buffer = await loadImageFromUrl(input)
464 let image = sharp(buffer)
465
466 if (modulate) {
467 image = image.modulate(modulate)
468 }
469 if (sharpen) {
470 image = image.sharpen({ sigma: sharpen.sigma })
471 }
472 if (blur) {
473 image = image.blur(blur)
474 }
475 if (greyscale) {
476 image = image.greyscale()
477 }
478 if (negate) {
479 image = image.negate()
480 }
481
482 const outputBuffer = await image.png({ compressionLevel: 9 }).toBuffer()
483
484 const result = await this.assetsService.uploadFromBuffer(userId, outputBuffer, {
485 type: AssetType.ImageEdit,
486 mimeType: 'image/png',
487 filename: 'adjusted.png',
488 }, 'image-edit')
489
490 this.logger.debug({ url: result.url }, '[adjustImage] Completed')
491 return {
492 content: [
493 {
494 type: 'resource_link',
495 uri: result.url,
496 name: `Image`,
497 },
498 { type: 'text', text: `Image adjusted successfully. URL: ${result.url}` },
499 ],
500 }
501 },
502 this.aiAvailability,
503 )
504 }
505
506 private createGetImageMetadataTool(_userId: string, _userType: UserType) {
507 return wrapTool(
508 this.logger,
509 ImageEditToolName.GetImageMetadata,
510 `Get image metadata (dimensions, format, color space).
511
512 **Parameters:**
513 - input: Image URL
514
515 **Returns:**
516 - width, height: Image dimensions in pixels
517 - format: Image format (jpeg, png, webp, etc.)
518 - channels: Number of color channels (3 = RGB, 4 = RGBA)
519 - hasAlpha: Whether image has transparency
520
521 **Example:**
522 { "input": "https://example.com/image.png" }`,
523 getImageMetadataSchema.shape,
524 async ({ input }) => {
525 this.logger.debug({ input }, '[getImageMetadata] Starting')
526
527 const buffer = await loadImageFromUrl(input)
528 const metadata = await sharp(buffer).metadata()
529
530 const result = {
531 width: metadata.width,
532 height: metadata.height,
533 format: metadata.format,
534 channels: metadata.channels,
535 hasAlpha: metadata.hasAlpha,
536 space: metadata.space,
537 density: metadata.density,
538 }
539
540 this.logger.debug({ metadata: result }, '[getImageMetadata] Completed')
541
542 return successResult(`Image Metadata:
543 - Dimensions: ${result.width}x${result.height} pixels
544 - Format: ${result.format}
545 - Channels: ${result.channels} (${result.hasAlpha ? 'with alpha' : 'no alpha'})
546 - Color Space: ${result.space || 'unknown'}
547 - Density: ${result.density || 'unknown'} DPI`)
548 },
549 this.aiAvailability,
550 )
551 }
552
553 createServer(userId: string, userType: UserType): McpSdkServerConfigWithInstance {
554 return createSdkMcpServer({
555 name: McpServerName.ImageEdit,
556 version: '1.0.0',
557 tools: [
558 this.createCompositeImagesTool(userId, userType),
559 this.createResizeImageTool(userId, userType),
560 this.createTransformImageTool(userId, userType),
561 this.createAdjustImageTool(userId, userType),
562 this.createGetImageMetadataTool(userId, userType),
563 ],
564 })
565 }
566 }
567
567 lines TYPESCRIPT