chore(unity): 固定Unity CLI Loop本地包并移除UnitySkills依赖

This commit is contained in:
Liubing\LB
2026-09-28 18:14:59 +08:00
parent 0eb43b2b40
commit c4ae02def9
952 changed files with 70137 additions and 11 deletions

View File

@@ -0,0 +1,61 @@
#nullable enable
using System.Collections.Generic;
namespace io.github.hatayama.uLoopMCP
{
public class ScreenshotInfo
{
public string ImagePath { get; set; } = "";
public long FileSizeBytes { get; set; }
public int Width { get; set; }
public int Height { get; set; }
// "gameView": image from game RenderTexture. Convert to simulate-mouse coords:
// sim_x = image_x / ResolutionScale
// sim_y = image_y / ResolutionScale + YOffset
// "window": EditorWindow capture including toolbar
public string CoordinateSystem { get; set; } = McpConstants.COORDINATE_SYSTEM_WINDOW;
public float ResolutionScale { get; set; } = 1.0f;
// Y offset to add to image pixel Y to get simulate-mouse Y coordinate.
// Only meaningful when CoordinateSystem == "gameView".
public int YOffset { get; set; }
public List<UIElementInfo> AnnotatedElements { get; set; } = new();
public ScreenshotInfo(string imagePath, long fileSizeBytes, int width, int height,
string coordinateSystem = McpConstants.COORDINATE_SYSTEM_WINDOW,
float resolutionScale = 1.0f, int yOffset = 0)
{
ImagePath = imagePath;
FileSizeBytes = fileSizeBytes;
Width = width;
Height = height;
CoordinateSystem = coordinateSystem;
ResolutionScale = resolutionScale;
YOffset = yOffset;
}
public ScreenshotInfo()
{
}
}
public class ScreenshotResponse : BaseToolResponse
{
public List<ScreenshotInfo> Screenshots { get; set; } = new();
public int ScreenshotCount => Screenshots.Count;
public ScreenshotResponse(List<ScreenshotInfo> screenshots)
{
Screenshots = screenshots;
}
public ScreenshotResponse()
{
}
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 9c8241beb9ee24fb79942b75fbee8524
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,55 @@
using System.ComponentModel;
namespace io.github.hatayama.uLoopMCP
{
/// <summary>
/// Window name matching modes for FindWindowsByName
/// </summary>
public enum WindowMatchMode
{
/// <summary>
/// Exact match (case-insensitive)
/// </summary>
exact = 0,
/// <summary>
/// Prefix match (case-insensitive)
/// </summary>
prefix = 1,
/// <summary>
/// Contains match (case-insensitive)
/// </summary>
contains = 2
}
public enum CaptureMode
{
window = 0,
rendering = 1
}
public class ScreenshotSchema : BaseToolSchema
{
[Description("Window name to capture (e.g., 'Game', 'Scene', 'Console', 'Inspector', 'Project', 'Hierarchy', or any EditorWindow title). Ignored when CaptureMode is rendering.")]
public string WindowName { get; set; } = "Game";
[Description("Resolution scale multiplier (0.1 to 1.0, where 1.0 is original size)")]
public float ResolutionScale { get; set; } = 1.0f;
[Description("Window name matching mode: exact(0)=exact match, prefix(1)=starts with, contains(2)=partial match. All modes are case-insensitive. Ignored when CaptureMode is rendering.")]
public WindowMatchMode MatchMode { get; set; } = WindowMatchMode.exact;
[Description("Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths.")]
public string OutputDirectory { get; set; } = "";
[Description("Capture mode: window(0)=capture EditorWindow including toolbar, rendering(1)=capture game rendering only (PlayMode required, coordinates match simulate-mouse)")]
public CaptureMode CaptureMode { get; set; } = CaptureMode.window;
[Description("Annotate interactive UI elements with names and simulate-mouse coordinates on the screenshot. Only works with CaptureMode=rendering in PlayMode.")]
public bool AnnotateElements { get; set; } = false;
[Description("Return only annotated element JSON without capturing a screenshot image. Requires AnnotateElements=true and CaptureMode=rendering in PlayMode.")]
public bool ElementsOnly { get; set; } = false;
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 5d23b680fbac84139986f03aabaca0b5
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,303 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
using UnityEditor;
namespace io.github.hatayama.uLoopMCP
{
[McpTool(Description = "Take a screenshot of Unity EditorWindow and save as PNG")]
public class ScreenshotTool : AbstractUnityTool<ScreenshotSchema, ScreenshotResponse>
{
private const int ANNOTATION_OVERLAY_RENDER_WAIT_FRAMES = 2;
public override string ToolName => "screenshot";
protected override async Task<ScreenshotResponse> ExecuteAsync(
ScreenshotSchema parameters,
CancellationToken ct)
{
ct.ThrowIfCancellationRequested();
string correlationId = McpConstants.GenerateCorrelationId();
VibeLogger.LogInfo(
"screenshot_start",
"Unity window screenshot started",
new { WindowName = parameters.WindowName, ResolutionScale = parameters.ResolutionScale, MatchMode = parameters.MatchMode.ToString(), OutputDirectory = parameters.OutputDirectory },
correlationId: correlationId,
humanNote: "User requested Unity window screenshot",
aiTodo: "Monitor capture performance and file size"
);
ValidateParameters(parameters);
if (parameters.CaptureMode == CaptureMode.rendering)
{
return await CaptureRenderingAsync(parameters, correlationId, ct);
}
return await CaptureWindowsAsync(parameters, correlationId, ct);
}
private async Task<ScreenshotResponse> CaptureRenderingAsync(
ScreenshotSchema parameters, string correlationId, CancellationToken ct)
{
if (!EditorApplication.isPlaying)
{
VibeLogger.LogError(
"screenshot_rendering_requires_playmode",
"CaptureMode.rendering requires PlayMode",
correlationId: correlationId
);
return new ScreenshotResponse();
}
List<UIElementInfo> annotatedElements = new List<UIElementInfo>();
if (parameters.AnnotateElements)
{
annotatedElements = UIElementAnnotator.CollectInteractiveElements();
UIElementAnnotator.AssignLabels(annotatedElements);
}
if (parameters.ElementsOnly)
{
UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y);
ScreenshotInfo elementsOnlyInfo = new ScreenshotInfo();
elementsOnlyInfo.CoordinateSystem = McpConstants.COORDINATE_SYSTEM_GAME_VIEW;
elementsOnlyInfo.AnnotatedElements = annotatedElements;
return new ScreenshotResponse(new List<ScreenshotInfo> { elementsOnlyInfo });
}
GameObject annotationOverlay = null;
Texture2D texture;
int yOffset;
try
{
if (parameters.AnnotateElements)
{
annotationOverlay = UIElementAnnotator.CreateAnnotationOverlay(
annotatedElements,
parameters.ResolutionScale);
Canvas.ForceUpdateCanvases();
// Chained CLI calls can read the previous GameView RT before overlay rendering catches up.
await EditorDelay.DelayFrame(ANNOTATION_OVERLAY_RENDER_WAIT_FRAMES, ct);
}
(texture, yOffset) = await EditorWindowCaptureUtility.CaptureGameRenderingAsync(
parameters.ResolutionScale, ct);
}
finally
{
UIElementAnnotator.DestroyAnnotationOverlay(annotationOverlay);
}
UIElementAnnotator.ConvertToSimCoordinates(annotatedElements, (int)Handles.GetMainGameViewSize().y);
if (texture == null)
{
VibeLogger.LogError(
"screenshot_rendering_unavailable",
"GameView RenderTexture is not available. Open the Game view and wait for a frame before retrying.",
correlationId: correlationId
);
return new ScreenshotResponse();
}
int width = texture.width;
int height = texture.height;
List<ScreenshotInfo> screenshots = new List<ScreenshotInfo>();
try
{
string outputDirectory = EnsureOutputDirectoryExists(parameters.OutputDirectory);
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss_fff");
string savedPath = Path.Combine(outputDirectory, $"Rendering_{timestamp}.png");
SaveTextureAsPng(texture, savedPath);
FileInfo savedFileInfo = new FileInfo(savedPath);
ScreenshotInfo info = new ScreenshotInfo(
savedPath, savedFileInfo.Length, width, height,
McpConstants.COORDINATE_SYSTEM_GAME_VIEW, parameters.ResolutionScale, yOffset);
info.AnnotatedElements = annotatedElements;
screenshots.Add(info);
}
catch (Exception ex)
{
// File I/O is external resource access; catch to report save failure
VibeLogger.LogWarning(
"screenshot_save_exception",
$"Exception saving rendering screenshot: {ex.Message}",
correlationId: correlationId
);
}
finally
{
UnityEngine.Object.DestroyImmediate(texture);
}
if (screenshots.Count > 0)
{
VibeLogger.LogInfo(
"screenshot_success",
$"Captured game rendering ({width}x{height})",
new { CaptureMode = "rendering", ScreenshotCount = screenshots.Count, AnnotatedElements = annotatedElements.Count },
correlationId: correlationId
);
}
return new ScreenshotResponse(screenshots);
}
private async Task<ScreenshotResponse> CaptureWindowsAsync(
ScreenshotSchema parameters, string correlationId, CancellationToken ct)
{
EditorWindow[] windows = EditorWindowCaptureUtility.FindWindowsByName(parameters.WindowName, parameters.MatchMode);
if (windows.Length == 0)
{
VibeLogger.LogError(
"screenshot_window_not_found",
$"Window '{parameters.WindowName}' not found (MatchMode: {parameters.MatchMode})",
correlationId: correlationId
);
return new ScreenshotResponse();
}
string outputDirectory = EnsureOutputDirectoryExists(parameters.OutputDirectory);
string safeWindowName = SanitizeFileName(parameters.WindowName);
string timestamp = DateTime.Now.ToString("yyyyMMdd_HHmmss_fff");
List<ScreenshotInfo> screenshots = new List<ScreenshotInfo>();
for (int i = 0; i < windows.Length; i++)
{
EditorWindow window = windows[i];
Texture2D texture = await EditorWindowCaptureUtility.CaptureWindowAsync(window, parameters.ResolutionScale, ct);
if (texture == null)
{
VibeLogger.LogWarning(
"screenshot_failed",
$"Failed to capture window index {i}",
correlationId: correlationId
);
continue;
}
string fileName = windows.Length == 1
? $"{safeWindowName}_{timestamp}.png"
: $"{safeWindowName}_{i + 1}_{timestamp}.png";
string savedPath = Path.Combine(outputDirectory, fileName);
int width = texture.width;
int height = texture.height;
try
{
SaveTextureAsPng(texture, savedPath);
FileInfo savedFileInfo = new FileInfo(savedPath);
screenshots.Add(new ScreenshotInfo(savedPath, savedFileInfo.Length, width, height));
}
catch (Exception ex)
{
// File I/O is external resource access; catch to continue processing remaining windows
VibeLogger.LogWarning(
"screenshot_save_exception",
$"Exception saving window index {i}: {ex.Message}",
correlationId: correlationId
);
}
finally
{
UnityEngine.Object.DestroyImmediate(texture);
}
}
VibeLogger.LogInfo(
"screenshot_success",
$"Captured {screenshots.Count} window(s)",
new { WindowName = parameters.WindowName, ScreenshotCount = screenshots.Count },
correlationId: correlationId
);
return new ScreenshotResponse(screenshots);
}
private void ValidateParameters(ScreenshotSchema parameters)
{
if (parameters.CaptureMode != CaptureMode.rendering &&
string.IsNullOrEmpty(parameters.WindowName))
{
throw new ParameterValidationException("WindowName cannot be null or empty");
}
if (parameters.ResolutionScale < 0.1f || parameters.ResolutionScale > 1.0f)
{
throw new ParameterValidationException(
$"ResolutionScale must be between 0.1 and 1.0, got: {parameters.ResolutionScale}");
}
// AnnotateElements and ElementsOnly rely on PlayMode rendering pipeline
if (parameters.CaptureMode != CaptureMode.rendering)
{
if (parameters.AnnotateElements)
{
throw new ParameterValidationException("AnnotateElements is only supported when CaptureMode=rendering");
}
if (parameters.ElementsOnly)
{
throw new ParameterValidationException("ElementsOnly is only supported when CaptureMode=rendering");
}
}
if (parameters.ElementsOnly && !parameters.AnnotateElements)
{
throw new ParameterValidationException("ElementsOnly requires AnnotateElements=true");
}
}
private string EnsureOutputDirectoryExists(string outputDirectory)
{
string resolvedDirectory;
if (string.IsNullOrEmpty(outputDirectory))
{
string projectRoot = UnityMcpPathResolver.GetProjectRoot();
resolvedDirectory = Path.Combine(projectRoot, McpConstants.OUTPUT_ROOT_DIR, McpConstants.SCREENSHOTS_DIR);
}
else
{
resolvedDirectory = Path.GetFullPath(outputDirectory);
}
Directory.CreateDirectory(resolvedDirectory);
return resolvedDirectory;
}
private string SanitizeFileName(string name)
{
char[] invalidChars = Path.GetInvalidFileNameChars();
string sanitized = name;
foreach (char c in invalidChars)
{
sanitized = sanitized.Replace(c, '_');
}
return sanitized;
}
private void SaveTextureAsPng(Texture2D texture, string fullPath)
{
byte[] pngData = texture.EncodeToPNG();
if (pngData == null)
{
throw new InvalidOperationException($"Failed to encode texture to PNG. Format: {texture.format}, Size: {texture.width}x{texture.height}");
}
File.WriteAllBytes(fullPath, pngData);
}
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 0d9887c6dbf3a4e5d8c087a9b1897207
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: e4781fead740e4fcb90a47ed4b2d72e6
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,96 @@
---
name: uloop-screenshot
description: "Capture Unity Editor windows or Game View rendering as PNG. Use for visual checks, debugging, documentation, or annotated UI element coordinates."
---
# uloop screenshot
Take a screenshot of any Unity EditorWindow by name and save as PNG.
## Usage
```bash
uloop screenshot [--window-name <name>] [--resolution-scale <scale>] [--match-mode <mode>] [--capture-mode <mode>] [--annotate-elements] [--elements-only] [--output-directory <path>]
```
## Parameters
| Parameter | Type | Default | Description |
|-----------|------|---------|-------------|
| `--window-name` | string | `Game` | Window name to capture. Ignored when `--capture-mode rendering`. |
| `--resolution-scale` | number | `1.0` | Resolution scale (0.1 to 1.0) |
| `--match-mode` | enum | `exact` | Window name matching mode: `exact`, `prefix`, or `contains`. Ignored when `--capture-mode rendering`. |
| `--capture-mode` | enum | `window` | `window`=capture EditorWindow including toolbar, `rendering`=capture game rendering only (PlayMode required, coordinates match simulate-mouse) |
| `--output-directory` | string | `""` | Output directory path for saving screenshots. When empty, uses default path (.uloop/outputs/Screenshots/). Accepts absolute paths. |
| `--annotate-elements` | boolean | `false` | Annotate interactive UI elements with index labels and interaction hints (A / CLICK, B / DRAG, ...). Only works with `--capture-mode rendering` in PlayMode. |
| `--elements-only` | boolean | `false` | Return only annotated element JSON without capturing a screenshot image. Requires `--annotate-elements` and `--capture-mode rendering` in PlayMode. |
## Match Modes
| Mode | Description | Example |
|------|-------------|---------|
| `exact` | Window name must match exactly (case-insensitive) | "Project" matches "Project" only |
| `prefix` | Window name must start with the input | "Project" matches "Project" and "Project Settings" |
| `contains` | Window name must contain the input anywhere | "set" matches "Project Settings" |
## Window Name
The window name is the text displayed in the window's title bar (tab). Common names: Game, Scene, Console, Inspector, Project, Hierarchy, Animation, Animator, Profiler. Custom EditorWindow titles are also supported.
## Global Options
| Option | Description |
|--------|-------------|
| `--project-path <path>` | Optional. Use only when the target Unity project is not the current directory. |
## Examples
```bash
# Take a screenshot of Game View (default)
uloop screenshot
# Capture game rendering (coordinates match simulate-mouse, PlayMode required)
uloop screenshot --capture-mode rendering
# Annotate interactive UI elements with index labels (for simulate-mouse workflow)
uloop screenshot --capture-mode rendering --annotate-elements
# Get UI element coordinates without capturing an image (fastest)
uloop screenshot --capture-mode rendering --annotate-elements --elements-only
# Take a screenshot of Scene View
uloop screenshot --window-name Scene
# Capture all windows starting with "Project" (prefix match)
uloop screenshot --window-name Project --match-mode prefix
# Save screenshot to a specific directory
uloop screenshot --output-directory /tmp/screenshots
# Combine options
uloop screenshot --window-name Scene --resolution-scale 0.5 --output-directory /tmp/screenshots
```
## Output
Returns JSON with:
- `ScreenshotCount`: Number of windows captured
- `Screenshots`: Array of screenshot info, each containing:
- `ImagePath`: Absolute path to the saved PNG file. Empty when `--elements-only` is used because no image file is written.
- `FileSizeBytes`: Size of the saved file in bytes
- `Width`: Captured image width in pixels
- `Height`: Captured image height in pixels
- `CoordinateSystem`: `"gameView"` or `"window"`
- `ResolutionScale`: Resolution scale used for capture
- `YOffset`: Y offset used for gameView coordinate conversion
- `AnnotatedElements`: Array of annotated UI element metadata. Empty unless `--annotate-elements` is used.
For `AnnotatedElements` fields and gameView coordinate conversion, read [references/annotated-elements.md](references/annotated-elements.md) before using screenshot coordinates with mouse simulation tools.
When multiple windows match (e.g., multiple Inspector windows or when using `contains` mode), all matching windows are captured with numbered filenames (e.g., `Inspector_1_*.png`, `Inspector_2_*.png`).
## Notes
- Use `uloop focus-window` first if needed
- Target window must be open in Unity Editor
- Window name matching is always case-insensitive

View File

@@ -0,0 +1,7 @@
fileFormatVersion: 2
guid: d3ca17bcf5b6546a6bde9c8958526273
TextScriptImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,8 @@
fileFormatVersion: 2
guid: 7a0b17bb8a40647d082146addab9e183
folderAsset: yes
DefaultImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,37 @@
# Annotated Elements and Coordinates
Read this when using `uloop screenshot --capture-mode rendering --annotate-elements` to find coordinates for `simulate-mouse-ui` or `simulate-mouse-input`.
## AnnotatedElements Fields
`AnnotatedElements` is empty unless `--annotate-elements` is used. Entries are sorted by z-order, frontmost first. Each item contains:
- `Label`: Index label in JSON (`A` = frontmost, `B` = next, ...). Screenshot labels also include the interaction hint, such as `A / CLICK` or `B / DRAG`.
- `Name`: Element name
- `Path`: Hierarchy path from the scene root, for example `Canvas/Panel/Button`. Use this as `simulate-mouse-ui --target-path` when bypassing raycast blockers.
- `Type`: Element type (`Button`, `Toggle`, `Slider`, `Dropdown`, `InputField`, `Scrollbar`, `Draggable`, `DropTarget`, `Selectable`)
- `Interaction`: Derived interaction category (`Click`, `Drag`, `Drop`, `Text`). Use this to choose between `simulate-mouse-ui --action Click` and drag actions.
- `SimX`, `SimY`: Center position in simulate-mouse coordinates. Use these directly with `--x` and `--y`.
- `BoundsMinX`, `BoundsMinY`, `BoundsMaxX`, `BoundsMaxY`: Bounding box in simulate-mouse coordinates
- `SortingOrder`: Canvas sorting order. Higher values are in front.
- `SiblingIndex`: Transform sibling index under the element's direct parent. Do not use it as a reliable z-order signal across nested UI hierarchies.
## Coordinate Conversion
When `CoordinateSystem` is `"gameView"`, convert image pixel coordinates to simulate-mouse coordinates:
```text
sim_x = image_x / ResolutionScale
sim_y = image_y / ResolutionScale + YOffset
```
When `ResolutionScale` is `1.0`, this simplifies to:
```text
sim_x = image_x
sim_y = image_y + YOffset
```
## Annotation Readability
Annotated screenshots compensate border thickness for `ResolutionScale`, so the saved PNG keeps the intended outline width after downscaling. The neutral contrast borders are 2 output pixels each, and the colored middle border is 4 output pixels. Label outlines are also compensated and are separated from element borders by a 4 output pixel gap.

View File

@@ -0,0 +1,7 @@
fileFormatVersion: 2
guid: 94b762d197c7747f997045586f70295e
TextScriptImporter:
externalObjects: {}
userData:
assetBundleName:
assetBundleVariant:

View File

@@ -0,0 +1,19 @@
namespace io.github.hatayama.uLoopMCP
{
public class UIElementInfo
{
public string Name { get; set; } = "";
public string Path { get; set; } = "";
public string Type { get; set; } = "";
public string Interaction { get; set; } = "";
public float SimX { get; set; }
public float SimY { get; set; }
public float BoundsMinX { get; set; }
public float BoundsMinY { get; set; }
public float BoundsMaxX { get; set; }
public float BoundsMaxY { get; set; }
public string Label { get; set; } = "";
public int SortingOrder { get; set; }
public int SiblingIndex { get; set; }
}
}

View File

@@ -0,0 +1,11 @@
fileFormatVersion: 2
guid: 35ace17262f7d4dc2b01041a3bdbe33a
MonoImporter:
externalObjects: {}
serializedVersion: 2
defaultReferences: []
executionOrder: 0
icon: {instanceID: 0}
userData:
assetBundleName:
assetBundleVariant: