chore(unity): 固定Unity CLI Loop本地包并移除UnitySkills依赖
This commit is contained in:
@@ -0,0 +1,38 @@
|
||||
#nullable enable
|
||||
using UnityEditor;
|
||||
using UnityEngine;
|
||||
using UnityEngine.EventSystems;
|
||||
|
||||
namespace io.github.hatayama.uLoopMCP
|
||||
{
|
||||
// Tool instances are created fresh per invocation, so drag state
|
||||
// between DragStart/DragMove/DragEnd must be held statically.
|
||||
[InitializeOnLoad]
|
||||
internal static class MouseDragState
|
||||
{
|
||||
internal static bool IsDragging => Target != null && PointerData != null;
|
||||
internal static GameObject? Target { get; set; }
|
||||
internal static PointerEventData? PointerData { get; set; }
|
||||
|
||||
static MouseDragState()
|
||||
{
|
||||
// PlayMode exit leaves dangling references to destroyed GameObjects
|
||||
EditorApplication.playModeStateChanged += OnPlayModeStateChanged;
|
||||
}
|
||||
|
||||
internal static void Clear()
|
||||
{
|
||||
Target = null;
|
||||
PointerData = null;
|
||||
}
|
||||
|
||||
private static void OnPlayModeStateChanged(PlayModeStateChange state)
|
||||
{
|
||||
if (state == PlayModeStateChange.ExitingPlayMode)
|
||||
{
|
||||
Clear();
|
||||
SimulateMouseUiOverlayState.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: df7edbb6ffd7849879330385aded44fc
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,20 @@
|
||||
#nullable enable
|
||||
|
||||
namespace io.github.hatayama.uLoopMCP
|
||||
{
|
||||
public class SimulateMouseUiResponse : BaseToolResponse
|
||||
{
|
||||
public bool Success { get; set; }
|
||||
public string Message { get; set; } = "";
|
||||
public string Action { get; set; } = "";
|
||||
public string? HitGameObjectName { get; set; }
|
||||
public float PositionX { get; set; }
|
||||
public float PositionY { get; set; }
|
||||
public float? EndPositionX { get; set; }
|
||||
public float? EndPositionY { get; set; }
|
||||
|
||||
public SimulateMouseUiResponse()
|
||||
{
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 82e2664246d214defa3afbab6c7a53c4
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,40 @@
|
||||
using System.ComponentModel;
|
||||
|
||||
namespace io.github.hatayama.uLoopMCP
|
||||
{
|
||||
public class SimulateMouseUiSchema : BaseToolSchema
|
||||
{
|
||||
[Description("Mouse action: Click(0) - click with selected Button, Drag(1) - one-shot drag, DragStart(2) - begin drag and hold, DragMove(3) - move while holding drag, DragEnd(4) - release drag, LongPress(5) - press and hold for Duration seconds")]
|
||||
public MouseAction Action { get; set; } = MouseAction.Click;
|
||||
|
||||
[Description("Target X position in screen pixels (origin: top-left). For Drag action, this is the destination.")]
|
||||
public float X { get; set; } = 0f;
|
||||
|
||||
[Description("Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination.")]
|
||||
public float Y { get; set; } = 0f;
|
||||
|
||||
[Description("Start X position in screen pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")]
|
||||
public float FromX { get; set; } = 0f;
|
||||
|
||||
[Description("Start Y position in screen pixels for Drag action (origin: top-left). Drag starts here and moves to X,Y.")]
|
||||
public float FromY { get; set; } = 0f;
|
||||
|
||||
[Description("Drag speed in pixels per second (0 for instant). Applies to Drag, DragMove, and DragEnd actions.")]
|
||||
public float DragSpeed { get; set; } = McpConstants.SIMULATE_MOUSE_UI_DEFAULT_DRAG_SPEED;
|
||||
|
||||
[Description("Hold duration in seconds for LongPress action (default: 0.5).")]
|
||||
public float Duration { get; set; } = 0.5f;
|
||||
|
||||
[Description("Mouse button: Left(0, default), Right(1), Middle(2).")]
|
||||
public MouseButton Button { get; set; } = MouseButton.Left;
|
||||
|
||||
[Description("Bypass EventSystem raycast and send click, long-press, or drag events directly to TargetPath, or DropTargetPath for DragEnd. Useful for interacting with UI behind a raycast-blocking overlay.")]
|
||||
public bool BypassRaycast { get; set; } = false;
|
||||
|
||||
[Description("Hierarchy path of the target GameObject used by Click, LongPress, Drag, and DragStart when BypassRaycast is true, for example Canvas/Panel/Button.")]
|
||||
public string TargetPath { get; set; } = "";
|
||||
|
||||
[Description("Optional hierarchy path of the drop target used by Drag and DragEnd, for example Canvas/DropZone.")]
|
||||
public string DropTargetPath { get; set; } = "";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 4bcfc2bdb0f824df18548ef6aa8e1f1b
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: b9fddf33ddb394d2684bd58ef4ec63f2
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,8 @@
|
||||
fileFormatVersion: 2
|
||||
guid: aa6c4215bedfe42e78bb59e626c88c00
|
||||
folderAsset: yes
|
||||
DefaultImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,132 @@
|
||||
---
|
||||
name: uloop-simulate-mouse-ui
|
||||
description: "Simulate PlayMode EventSystem UI mouse actions using screen coordinates. Use for UI clicks, long-presses, or drags from annotated screenshots."
|
||||
context: fork
|
||||
---
|
||||
|
||||
# Task
|
||||
|
||||
Simulate mouse interaction on Unity PlayMode UI: $ARGUMENTS
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Ensure Unity is in PlayMode (use `uloop control-play-mode --action Play` if not)
|
||||
2. Get UI element info: `uloop screenshot --capture-mode rendering --annotate-elements --elements-only`
|
||||
3. Use the `AnnotatedElements` array to find the target element by `Label`, `Name`, or `Path` (A=frontmost, B=next, ...). Use `Interaction` to distinguish click targets from drag/drop/text targets, then use `SimX`/`SimY` directly as `--x`/`--y` coordinates.
|
||||
4. Execute the appropriate `uloop simulate-mouse-ui` command
|
||||
5. Take a screenshot to verify the result: `uloop screenshot --capture-mode rendering --annotate-elements`
|
||||
6. Report what happened
|
||||
|
||||
## Tool Reference
|
||||
|
||||
```bash
|
||||
uloop simulate-mouse-ui --action <action> --x <x> --y <y> [options]
|
||||
```
|
||||
|
||||
### Parameters
|
||||
|
||||
| Parameter | Type | Default | Description |
|
||||
|-----------|------|---------|-------------|
|
||||
| `--action` | enum | `Click` | `Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, `LongPress` |
|
||||
| `--x` | number | `0` | Target X position in screen pixels (origin: top-left). For Drag action, this is the destination. |
|
||||
| `--y` | number | `0` | Target Y position in screen pixels (origin: top-left). For Drag action, this is the destination. |
|
||||
| `--from-x` | number | `0` | Start X position for Drag action. Drag starts here and moves to x,y. |
|
||||
| `--from-y` | number | `0` | Start Y position for Drag action. Drag starts here and moves to x,y. |
|
||||
| `--drag-speed` | number | `2000` | Drag speed in pixels per second (0 for instant). 2000 is fast (default), 200 is slow enough to watch. Applies to Drag, DragMove, and DragEnd actions. |
|
||||
| `--duration` | number | `0.5` | Hold duration in seconds for LongPress action. |
|
||||
| `--button` | enum | `Left` | Mouse button. `Click` and `LongPress` support `Left`, `Right`, and `Middle`. Drag actions support `Left` only; other buttons return an error. |
|
||||
| `--bypass-raycast` | boolean | `false` | For `Click`, `LongPress`, `Drag`, and `DragStart`, bypass EventSystem raycast and dispatch pointer events directly to `--target-path`. Use when a raycast-blocking overlay visually covers the intended target. |
|
||||
| `--target-path` | string | `""` | Hierarchy path of the target GameObject, for example `Canvas/Panel/Button`. Required when `--bypass-raycast true` is used with `Click`, `LongPress`, `Drag`, or `DragStart`; prefer `AnnotatedElements[].Path` from screenshot JSON. |
|
||||
| `--drop-target-path` | string | `""` | Optional hierarchy path of a drop target for `Drag` or `DragEnd`, for example `Canvas/DropZone`. Use this when the drop zone is also behind a raycast blocker. |
|
||||
|
||||
### Actions
|
||||
|
||||
| Action | Event Fired | Description |
|
||||
|--------|-------------|-------------|
|
||||
| `Click` | PointerDown → PointerUp → PointerClick | Click at (x, y) with the selected `--button` |
|
||||
| `LongPress` | PointerDown → (hold) → PointerUp | Press and hold at (x, y) for `--duration` seconds, then release. No PointerClick is fired. |
|
||||
| `Drag` | BeginDrag → Drag×N → EndDrag | One-shot drag from (fromX, fromY) to (x, y) at the specified speed |
|
||||
| `DragStart` | BeginDrag | Begin drag at (x, y) and hold |
|
||||
| `DragMove` | Drag×N | Animate from current position to (x, y) at the specified speed |
|
||||
| `DragEnd` | Drag×N → EndDrag | Animate to (x, y) at the specified speed, then release drag |
|
||||
|
||||
### Split Drag Rules
|
||||
|
||||
- `DragStart` must be called before `DragMove` or `DragEnd`
|
||||
- `DragEnd` must be called to release an active drag — failing to call it leaves drag state stuck
|
||||
- Calling `DragMove` or `DragEnd` without an active drag returns an error
|
||||
- `Drag`, `DragStart`, `DragMove`, and `DragEnd` only support `--button Left`
|
||||
|
||||
### Global Options (all optional, mutually exclusive)
|
||||
|
||||
| Option | Description |
|
||||
|--------|-------------|
|
||||
| `--project-path <path>` | Optional. Use only when the target Unity project is not the current directory. |
|
||||
|
||||
|
||||
## Coordinate System
|
||||
|
||||
- Origin is **top-left** (0, 0)
|
||||
- All positions are in **screen pixels**
|
||||
- Get coordinates from `AnnotatedElements` JSON (`SimX`/`SimY`) — do NOT look up GameObject positions
|
||||
- Clicking or long-pressing on empty space (no UI element) still succeeds with a message indicating no element was hit
|
||||
- Dragging on empty space (no draggable UI element) returns `Success = false`
|
||||
- `--bypass-raycast true` still uses coordinates for pointer event positions, but chooses the clicked, long-pressed, or dragged GameObject by `--target-path`
|
||||
- If `--target-path` or `--drop-target-path` matches multiple active GameObjects, the command fails instead of choosing an arbitrary duplicate
|
||||
|
||||
## Examples
|
||||
|
||||
```bash
|
||||
# Click a button at screen position
|
||||
uloop simulate-mouse-ui --action Click --x 400 --y 300
|
||||
|
||||
# Force-click a button behind a raycast blocker by path
|
||||
uloop simulate-mouse-ui --action Click --x 400 --y 300 --bypass-raycast true --target-path "Canvas/Panel/Button"
|
||||
|
||||
# Force-long-press a button behind a raycast blocker by path
|
||||
uloop simulate-mouse-ui --action LongPress --x 400 --y 300 --duration 3.0 --bypass-raycast true --target-path "Canvas/Panel/Button"
|
||||
|
||||
# Force-drag an item behind a raycast blocker by path
|
||||
uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 --bypass-raycast true --target-path "Canvas/Item"
|
||||
|
||||
# Force-drag and dispatch Drop to a blocked drop zone
|
||||
uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 --bypass-raycast true --target-path "Canvas/Item" --drop-target-path "Canvas/DropZone"
|
||||
|
||||
# Long-press a button for 3 seconds
|
||||
uloop simulate-mouse-ui --action LongPress --x 400 --y 300 --duration 3.0
|
||||
|
||||
# One-shot drag (start to end in one call)
|
||||
uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300
|
||||
|
||||
# Slow drag for visual inspection
|
||||
uloop simulate-mouse-ui --action Drag --from-x 400 --from-y 300 --x 600 --y 300 --drag-speed 200
|
||||
|
||||
# Split drag with hold (for inspection between steps)
|
||||
uloop simulate-mouse-ui --action DragStart --x 400 --y 300
|
||||
uloop screenshot --window-name Game
|
||||
uloop simulate-mouse-ui --action DragMove --x 500 --y 300
|
||||
uloop simulate-mouse-ui --action DragEnd --x 600 --y 300
|
||||
```
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- Unity must be in **PlayMode**
|
||||
- Target scene must have an **EventSystem** GameObject
|
||||
- UI elements must have a **GraphicRaycaster** on their Canvas
|
||||
- If you need gameplay mouse input rather than UI pointer events, `simulate-mouse-input` assumes the project uses the New Input System; otherwise prefer `execute-dynamic-code`
|
||||
|
||||
## Output
|
||||
|
||||
Returns JSON with:
|
||||
- `Success`: Whether the operation succeeded
|
||||
- `Message`: Status message (e.g. "Hit element: ButtonStart" or "No UI element under (x, y)")
|
||||
- `Action`: Echoes which action was executed (`Click`, `Drag`, `DragStart`, `DragMove`, `DragEnd`, or `LongPress`)
|
||||
- `HitGameObjectName`: Name of the topmost UI element under the pointer (nullable string; null if nothing was hit)
|
||||
- `PositionX`: Target X coordinate that was used
|
||||
- `PositionY`: Target Y coordinate that was used
|
||||
- `EndPositionX`: Drag end X coordinate (nullable float; populated for drag actions only)
|
||||
- `EndPositionY`: Drag end Y coordinate (nullable float; populated for drag actions only)
|
||||
|
||||
These are the only eight fields. There is no `Button`, `Duration`, `DragSpeed`, raycast list, or pointer-event log in the response — verify the visual outcome with a follow-up `uloop screenshot --capture-mode rendering --annotate-elements`.
|
||||
|
||||
Note: Click and LongPress on empty space (no UI element) still return `Success = true` with `HitGameObjectName = null`. Drag actions on empty space return `Success = false`.
|
||||
@@ -0,0 +1,7 @@
|
||||
fileFormatVersion: 2
|
||||
guid: 91200dbd0b4d241c68687ca5a2ed5698
|
||||
TextScriptImporter:
|
||||
externalObjects: {}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
@@ -0,0 +1,193 @@
|
||||
#nullable enable
|
||||
using System.Collections.Generic;
|
||||
using UnityEngine;
|
||||
using UnityEngine.EventSystems;
|
||||
using UnityEngine.UI;
|
||||
|
||||
namespace io.github.hatayama.uLoopMCP
|
||||
{
|
||||
// Shared UI raycast logic used by SimulateMouseUiTool and InputReplayer.
|
||||
internal static class UiRaycastHelper
|
||||
{
|
||||
public static RaycastResult? RaycastUI(Vector2 screenPosition, EventSystem eventSystem)
|
||||
{
|
||||
PointerEventData pointerData = new PointerEventData(eventSystem)
|
||||
{
|
||||
position = screenPosition
|
||||
};
|
||||
List<RaycastResult> results = new List<RaycastResult>();
|
||||
eventSystem.RaycastAll(pointerData, results);
|
||||
RaycastResult? canvasSpaceHit = RaycastCanvasSpace(screenPosition);
|
||||
|
||||
if (results.Count > 0)
|
||||
{
|
||||
RaycastResult firstHit = results[0];
|
||||
|
||||
if (canvasSpaceHit != null && ShouldPreferCanvasSpaceHit(canvasSpaceHit.Value, firstHit))
|
||||
{
|
||||
return canvasSpaceHit;
|
||||
}
|
||||
|
||||
return firstHit;
|
||||
}
|
||||
|
||||
// EventSystem clips at Screen.width/height, which can be smaller than the
|
||||
// Canvas layout space (Game view target resolution). Fall back to manual hit testing.
|
||||
return canvasSpaceHit;
|
||||
}
|
||||
|
||||
// Bypass EventSystem's Screen-bounds clipping by directly testing Graphic rects in Canvas space.
|
||||
// Only supports ScreenSpaceOverlay canvases where world positions equal Canvas-space positions.
|
||||
public static RaycastResult? RaycastCanvasSpace(Vector2 canvasPosition)
|
||||
{
|
||||
Canvas[] canvases = Object.FindObjectsByType<Canvas>(FindObjectsSortMode.None);
|
||||
RaycastResult? bestHit = null;
|
||||
|
||||
foreach (Canvas canvas in canvases)
|
||||
{
|
||||
if (!canvas.gameObject.activeInHierarchy)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
if (canvas.renderMode != RenderMode.ScreenSpaceOverlay)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
GraphicRaycaster? raycaster = canvas.GetComponent<GraphicRaycaster>();
|
||||
if (raycaster == null || !raycaster.enabled)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
Graphic[] graphics = canvas.GetComponentsInChildren<Graphic>();
|
||||
foreach (Graphic graphic in graphics)
|
||||
{
|
||||
if (!IsRaycastCandidate(graphic, canvasPosition))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
RaycastResult candidate = new RaycastResult
|
||||
{
|
||||
gameObject = graphic.gameObject,
|
||||
module = raycaster,
|
||||
screenPosition = canvasPosition,
|
||||
sortingLayer = canvas.sortingLayerID,
|
||||
sortingOrder = canvas.sortingOrder,
|
||||
depth = graphic.depth
|
||||
};
|
||||
|
||||
if (bestHit == null || CompareRaycastPriority(candidate, bestHit.Value) > 0)
|
||||
{
|
||||
bestHit = candidate;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return bestHit;
|
||||
}
|
||||
|
||||
// Respects CanvasGroup.blocksRaycasts, Mask, RectMask2D, and custom ICanvasRaycastFilter
|
||||
private static bool IsRaycastCandidate(Graphic graphic, Vector2 canvasPosition)
|
||||
{
|
||||
if (!graphic.gameObject.activeInHierarchy || !graphic.enabled)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!graphic.raycastTarget || graphic.depth == -1 || graphic.canvasRenderer.cull)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!RectTransformUtility.RectangleContainsScreenPoint(
|
||||
graphic.rectTransform, canvasPosition, null))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
return graphic.Raycast(canvasPosition, null);
|
||||
}
|
||||
|
||||
private static bool ShouldPreferCanvasSpaceHit(RaycastResult canvasSpaceHit, RaycastResult eventSystemHit)
|
||||
{
|
||||
if (!IsGraphicRaycast(canvasSpaceHit))
|
||||
{
|
||||
return false;
|
||||
}
|
||||
|
||||
if (!IsGraphicRaycast(eventSystemHit))
|
||||
{
|
||||
return true;
|
||||
}
|
||||
|
||||
return CompareRaycastPriority(canvasSpaceHit, eventSystemHit) > 0;
|
||||
}
|
||||
|
||||
private static bool IsGraphicRaycast(RaycastResult raycastResult)
|
||||
{
|
||||
return raycastResult.module is GraphicRaycaster;
|
||||
}
|
||||
|
||||
private static int CompareRaycastPriority(RaycastResult left, RaycastResult right)
|
||||
{
|
||||
int sortOrderPriority = Compare(GetSortOrderPriority(left), GetSortOrderPriority(right));
|
||||
if (sortOrderPriority != 0)
|
||||
{
|
||||
return sortOrderPriority;
|
||||
}
|
||||
|
||||
int renderOrderPriority = Compare(GetRenderOrderPriority(left), GetRenderOrderPriority(right));
|
||||
if (renderOrderPriority != 0)
|
||||
{
|
||||
return renderOrderPriority;
|
||||
}
|
||||
|
||||
int sortingLayer = Compare(GetSortingLayerValue(left), GetSortingLayerValue(right));
|
||||
if (sortingLayer != 0)
|
||||
{
|
||||
return sortingLayer;
|
||||
}
|
||||
|
||||
int sortingOrder = Compare(left.sortingOrder, right.sortingOrder);
|
||||
if (sortingOrder != 0)
|
||||
{
|
||||
return sortingOrder;
|
||||
}
|
||||
|
||||
return Compare(left.depth, right.depth);
|
||||
}
|
||||
|
||||
private static int GetSortOrderPriority(RaycastResult result)
|
||||
{
|
||||
return result.module != null ? result.module.sortOrderPriority : 0;
|
||||
}
|
||||
|
||||
private static int GetRenderOrderPriority(RaycastResult result)
|
||||
{
|
||||
return result.module != null ? result.module.renderOrderPriority : 0;
|
||||
}
|
||||
|
||||
private static int GetSortingLayerValue(RaycastResult result)
|
||||
{
|
||||
return SortingLayer.GetLayerValueFromID(result.sortingLayer);
|
||||
}
|
||||
|
||||
private static int Compare(int left, int right)
|
||||
{
|
||||
if (left > right)
|
||||
{
|
||||
return 1;
|
||||
}
|
||||
|
||||
if (left < right)
|
||||
{
|
||||
return -1;
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
fileFormatVersion: 2
|
||||
guid: b379f268312ef42708e85415f1aa4e98
|
||||
MonoImporter:
|
||||
externalObjects: {}
|
||||
serializedVersion: 2
|
||||
defaultReferences: []
|
||||
executionOrder: 0
|
||||
icon: {instanceID: 0}
|
||||
userData:
|
||||
assetBundleName:
|
||||
assetBundleVariant:
|
||||
Reference in New Issue
Block a user