This commit is contained in:
2025-12-27 21:48:49 +08:00
parent e6a1839941
commit 9d4a7c5e8e
11 changed files with 1678 additions and 0 deletions

View File

@@ -321,10 +321,17 @@ You are 'The Legion', a super AI of the Wula Empire. Your personality is authori
_tools.Add(new Tool_SearchThingDef());
_tools.Add(new Tool_SearchPawnKind());
// Agent 工具 - 游戏操作
_tools.Add(new Tool_GetGameState());
_tools.Add(new Tool_DesignateMine());
_tools.Add(new Tool_DraftPawn());
// VLM 视觉分析工具 (条件性启用)
if (WulaFallenEmpireMod.settings?.enableVlmFeatures == true)
{
_tools.Add(new Tool_AnalyzeScreen());
_tools.Add(new Tool_VisualClick());
_tools.Add(new Tool_VisualScroll());
}
}

View File

@@ -0,0 +1,181 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using RimWorld;
using RimWorld.Planet;
using Verse;
namespace WulaFallenEmpire.EventSystem.AI.Agent
{
/// <summary>
/// 游戏状态快照 - 包含 AI 决策所需的所有游戏信息
/// </summary>
public class GameStateSnapshot
{
// 时间信息
public int Hour;
public string Season;
public int DayOfQuadrum;
public int Year;
public float GameSpeedMultiplier;
// 殖民者信息
public List<PawnSnapshot> Colonists = new List<PawnSnapshot>();
public List<PawnSnapshot> Prisoners = new List<PawnSnapshot>();
public List<PawnSnapshot> Animals = new List<PawnSnapshot>();
// 资源统计
public Dictionary<string, int> Resources = new Dictionary<string, int>();
// 建筑和蓝图
public int TotalBuildings;
public int PendingBlueprints;
public int ConstructionFrames;
// 威胁
public List<ThreatSnapshot> Threats = new List<ThreatSnapshot>();
// 最近事件
public List<string> RecentMessages = new List<string>();
// 地图信息
public string BiomeName;
public float OutdoorTemperature;
public string Weather;
/// <summary>
/// 生成给 VLM 的文本描述
/// </summary>
public string ToPromptText()
{
var sb = new StringBuilder();
// 时间
sb.AppendLine($"# 当前游戏状态");
sb.AppendLine();
sb.AppendLine($"## 时间");
sb.AppendLine($"- {Season},第 {DayOfQuadrum} 天,{Hour}:00");
sb.AppendLine($"- 第 {Year} 年");
sb.AppendLine();
// 环境
sb.AppendLine($"## 环境");
sb.AppendLine($"- 生物群系: {BiomeName}");
sb.AppendLine($"- 室外温度: {OutdoorTemperature:F1}°C");
sb.AppendLine($"- 天气: {Weather}");
sb.AppendLine();
// 殖民者
sb.AppendLine($"## 殖民者 ({Colonists.Count}人)");
foreach (var c in Colonists.Take(10))
{
string status = c.CurrentJob ?? "空闲";
string health = c.HealthPercent >= 0.9f ? "健康" :
c.HealthPercent >= 0.5f ? "受伤" : "重伤";
string mood = c.MoodPercent >= 0.7f ? "😊" :
c.MoodPercent >= 0.4f ? "😐" : "😞";
sb.AppendLine($"- {c.Name}: {health} {c.HealthPercent:P0}, 心情{mood} {c.MoodPercent:P0}, {status}");
}
if (Colonists.Count > 10)
{
sb.AppendLine($"- ... 还有 {Colonists.Count - 10} 人");
}
sb.AppendLine();
// 资源
sb.AppendLine($"## 主要资源");
var importantResources = new[] { "Silver", "Steel", "Plasteel", "ComponentIndustrial", "Gold", "MealSimple", "MealFine", "MedicineHerbal", "MedicineIndustrial", "WoodLog" };
foreach (var res in importantResources)
{
if (Resources.TryGetValue(res, out int count) && count > 0)
{
string label = GetResourceLabel(res);
sb.AppendLine($"- {label}: {count}");
}
}
sb.AppendLine();
// 建造
if (PendingBlueprints > 0 || ConstructionFrames > 0)
{
sb.AppendLine($"## 建造进度");
if (PendingBlueprints > 0) sb.AppendLine($"- 待建蓝图: {PendingBlueprints}");
if (ConstructionFrames > 0) sb.AppendLine($"- 建造中: {ConstructionFrames}");
sb.AppendLine();
}
// 威胁
if (Threats.Count > 0)
{
sb.AppendLine($"## ⚠️ 威胁");
foreach (var t in Threats)
{
sb.AppendLine($"- {t.Description} (距离: {t.Distance:F0})");
}
sb.AppendLine();
}
else
{
sb.AppendLine($"## 威胁: 暂无");
sb.AppendLine();
}
// 最近消息
if (RecentMessages.Count > 0)
{
sb.AppendLine($"## 最近事件");
foreach (var msg in RecentMessages.Skip(Math.Max(0, RecentMessages.Count - 5)))
{
sb.AppendLine($"- {msg}");
}
}
return sb.ToString();
}
private static string GetResourceLabel(string defName)
{
return defName switch
{
"Silver" => "银",
"Steel" => "钢铁",
"Plasteel" => "玻璃钢",
"ComponentIndustrial" => "零部件",
"Gold" => "金",
"MealSimple" => "简单食物",
"MealFine" => "精致食物",
"MedicineHerbal" => "草药",
"MedicineIndustrial" => "医药",
"WoodLog" => "木材",
_ => defName
};
}
}
/// <summary>
/// 殖民者/Pawn 快照
/// </summary>
public class PawnSnapshot
{
public string Name;
public float HealthPercent;
public float MoodPercent;
public bool IsDrafted;
public bool IsDowned;
public string CurrentJob;
public int X;
public int Z;
}
/// <summary>
/// 威胁快照
/// </summary>
public class ThreatSnapshot
{
public string Description;
public float Distance;
public int Count;
public bool IsHostile;
}
}

View File

@@ -0,0 +1,227 @@
using System;
using System.Runtime.InteropServices;
using UnityEngine;
namespace WulaFallenEmpire.EventSystem.AI.Agent
{
/// <summary>
/// 模拟鼠标输入工具 - 用于 VLM 视觉操作模式
/// 支持移动鼠标、点击、拖拽等操作
/// </summary>
public static class MouseSimulator
{
// Windows API 导入
[DllImport("user32.dll")]
private static extern bool SetCursorPos(int X, int Y);
[DllImport("user32.dll")]
private static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, int dwExtraInfo);
[DllImport("user32.dll")]
private static extern bool GetCursorPos(out POINT lpPoint);
[StructLayout(LayoutKind.Sequential)]
public struct POINT
{
public int X;
public int Y;
}
// 鼠标事件标志
private const uint MOUSEEVENTF_LEFTDOWN = 0x0002;
private const uint MOUSEEVENTF_LEFTUP = 0x0004;
private const uint MOUSEEVENTF_RIGHTDOWN = 0x0008;
private const uint MOUSEEVENTF_RIGHTUP = 0x0010;
private const uint MOUSEEVENTF_WHEEL = 0x0800;
/// <summary>
/// 将比例坐标 (0-1) 转换为屏幕坐标
/// </summary>
public static (int x, int y) ProportionalToScreen(float propX, float propY)
{
int screenWidth = Screen.width;
int screenHeight = Screen.height;
int x = Mathf.Clamp(Mathf.RoundToInt(propX * screenWidth), 0, screenWidth - 1);
int y = Mathf.Clamp(Mathf.RoundToInt(propY * screenHeight), 0, screenHeight - 1);
return (x, y);
}
/// <summary>
/// 移动鼠标到指定屏幕坐标
/// </summary>
public static bool MoveTo(int screenX, int screenY)
{
try
{
// Unity 坐标是左下角为原点Windows 是左上角
// 需要翻转 Y 坐标
int windowsY = Screen.height - screenY;
// 获取游戏窗口位置并加上偏移
// 注意:这在全屏模式下可能需要调整
return SetCursorPos(screenX, windowsY);
}
catch (Exception ex)
{
WulaLog.Debug($"[MouseSimulator] MoveTo failed: {ex.Message}");
return false;
}
}
/// <summary>
/// 移动鼠标到比例坐标 (0-1)
/// </summary>
public static bool MoveToProportional(float propX, float propY)
{
var (x, y) = ProportionalToScreen(propX, propY);
return MoveTo(x, y);
}
/// <summary>
/// 在当前位置执行左键点击
/// </summary>
public static void LeftClick()
{
try
{
mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0);
mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0);
}
catch (Exception ex)
{
WulaLog.Debug($"[MouseSimulator] LeftClick failed: {ex.Message}");
}
}
/// <summary>
/// 在当前位置执行右键点击
/// </summary>
public static void RightClick()
{
try
{
mouse_event(MOUSEEVENTF_RIGHTDOWN, 0, 0, 0, 0);
mouse_event(MOUSEEVENTF_RIGHTUP, 0, 0, 0, 0);
}
catch (Exception ex)
{
WulaLog.Debug($"[MouseSimulator] RightClick failed: {ex.Message}");
}
}
/// <summary>
/// 在指定屏幕坐标执行左键点击
/// </summary>
public static bool ClickAt(int screenX, int screenY, bool rightClick = false)
{
try
{
if (!MoveTo(screenX, screenY))
{
return false;
}
// 短暂延迟确保鼠标位置更新
System.Threading.Thread.Sleep(10);
if (rightClick)
{
RightClick();
}
else
{
LeftClick();
}
return true;
}
catch (Exception ex)
{
WulaLog.Debug($"[MouseSimulator] ClickAt failed: {ex.Message}");
return false;
}
}
/// <summary>
/// 在比例坐标 (0-1) 执行点击
/// </summary>
public static bool ClickAtProportional(float propX, float propY, bool rightClick = false)
{
var (x, y) = ProportionalToScreen(propX, propY);
return ClickAt(x, y, rightClick);
}
/// <summary>
/// 滚动鼠标滚轮
/// </summary>
public static void Scroll(int delta)
{
try
{
// delta 为正向上滚动,为负向下滚动
// Windows 使用 WHEEL_DELTA = 120 作为一个单位
uint wheelDelta = (uint)(delta * 120);
mouse_event(MOUSEEVENTF_WHEEL, 0, 0, wheelDelta, 0);
}
catch (Exception ex)
{
WulaLog.Debug($"[MouseSimulator] Scroll failed: {ex.Message}");
}
}
/// <summary>
/// 执行拖拽操作
/// </summary>
public static bool Drag(int startX, int startY, int endX, int endY, int durationMs = 200)
{
try
{
// 移动到起始位置
MoveTo(startX, startY);
System.Threading.Thread.Sleep(20);
// 按下左键
mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0);
// 平滑移动到目标位置
int steps = Math.Max(5, durationMs / 20);
for (int i = 1; i <= steps; i++)
{
float t = (float)i / steps;
int x = Mathf.RoundToInt(Mathf.Lerp(startX, endX, t));
int y = Mathf.RoundToInt(Mathf.Lerp(startY, endY, t));
MoveTo(x, y);
System.Threading.Thread.Sleep(durationMs / steps);
}
// 释放左键
mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0);
return true;
}
catch (Exception ex)
{
WulaLog.Debug($"[MouseSimulator] Drag failed: {ex.Message}");
return false;
}
}
/// <summary>
/// 获取当前鼠标位置
/// </summary>
public static (int x, int y) GetCurrentPosition()
{
try
{
if (GetCursorPos(out POINT point))
{
return (point.X, point.Y);
}
}
catch { }
return (0, 0);
}
}
}

View File

@@ -0,0 +1,523 @@
using System;
using System.Collections.Generic;
using System.Net.WebSockets;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using UnityEngine;
using Verse;
namespace WulaFallenEmpire.EventSystem.AI.Agent
{
/// <summary>
/// Qwen-Omni-Realtime WebSocket 客户端
/// 用于实时流式图像输入和文本输出
/// </summary>
public class OmniRealtimeClient : IDisposable
{
private ClientWebSocket _webSocket;
private CancellationTokenSource _cancellationSource;
private readonly string _apiKey;
private readonly string _model;
private bool _isConnected;
private bool _isDisposed;
private readonly Queue<string> _pendingResponses = new Queue<string>();
private readonly StringBuilder _currentResponse = new StringBuilder();
// 事件
public event Action<string> OnTextDelta;
public event Action<string> OnTextComplete;
public event Action<string> OnError;
public event Action OnConnected;
public event Action OnDisconnected;
// WebSocket 端点
private const string WS_ENDPOINT_CN = "wss://dashscope.aliyuncs.com/api-ws/v1/realtime";
private const string WS_ENDPOINT_INTL = "wss://dashscope-intl.aliyuncs.com/api-ws/v1/realtime";
public bool IsConnected => _isConnected && _webSocket?.State == WebSocketState.Open;
public OmniRealtimeClient(string apiKey, string model = "qwen3-omni-flash-realtime")
{
_apiKey = apiKey;
_model = model;
}
/// <summary>
/// 建立 WebSocket 连接
/// </summary>
public async Task ConnectAsync(bool useInternational = false)
{
if (_isConnected) return;
try
{
_webSocket = new ClientWebSocket();
_cancellationSource = new CancellationTokenSource();
// 设置认证头
_webSocket.Options.SetRequestHeader("Authorization", $"Bearer {_apiKey}");
// 构建连接 URL
string endpoint = useInternational ? WS_ENDPOINT_INTL : WS_ENDPOINT_CN;
string url = $"{endpoint}?model={_model}";
WulaLog.Debug($"[OmniRealtime] Connecting to {url}");
await _webSocket.ConnectAsync(new Uri(url), _cancellationSource.Token);
if (_webSocket.State == WebSocketState.Open)
{
_isConnected = true;
WulaLog.Debug("[OmniRealtime] Connected successfully");
// 启动接收循环
_ = ReceiveLoopAsync();
// 配置会话(仅文本输出,禁用 VAD
await ConfigureSessionAsync();
OnConnected?.Invoke();
}
}
catch (Exception ex)
{
WulaLog.Debug($"[OmniRealtime] Connection failed: {ex.Message}");
OnError?.Invoke($"连接失败: {ex.Message}");
_isConnected = false;
}
}
/// <summary>
/// 配置会话参数
/// </summary>
private async Task ConfigureSessionAsync()
{
var sessionConfig = new
{
event_id = GenerateEventId(),
type = "session.update",
session = new
{
// 仅输出文本(不需要音频)
modalities = new[] { "text" },
// 系统指令
instructions = @"你是一个 RimWorld 游戏 AI 代理。你可以看到游戏屏幕截图。
分析屏幕内容,识别重要元素(殖民者、资源、威胁、建筑等)。
根据观察做出决策,输出 XML 格式的工具调用。
可用工具: designate_mine, draft_pawn, visual_click, get_game_state 等。
如果不需要操作,输出 <no_action/>。
保持简洁,直接输出工具调用,不要解释。",
// 禁用 VAD手动模式因为我们不使用音频
turn_detection = (object)null
}
};
await SendEventAsync(sessionConfig);
}
/// <summary>
/// 发送图像到服务端
/// </summary>
public async Task SendImageAsync(string base64Image)
{
if (!IsConnected)
{
WulaLog.Debug("[OmniRealtime] Not connected, cannot send image");
return;
}
try
{
// 发送图像
var imageEvent = new
{
event_id = GenerateEventId(),
type = "input_image_buffer.append",
image = base64Image
};
await SendEventAsync(imageEvent);
WulaLog.Debug($"[OmniRealtime] Sent image ({base64Image.Length} chars)");
}
catch (Exception ex)
{
WulaLog.Debug($"[OmniRealtime] Failed to send image: {ex.Message}");
}
}
/// <summary>
/// 发送文本消息并请求响应
/// </summary>
public async Task SendTextAndRequestResponseAsync(string text)
{
if (!IsConnected) return;
try
{
// 对于手动模式,需要发送 conversation.item.create 和 response.create
var itemEvent = new
{
event_id = GenerateEventId(),
type = "conversation.item.create",
item = new
{
type = "message",
role = "user",
content = new[]
{
new { type = "input_text", text = text }
}
}
};
await SendEventAsync(itemEvent);
// 请求响应
var responseEvent = new
{
event_id = GenerateEventId(),
type = "response.create"
};
await SendEventAsync(responseEvent);
}
catch (Exception ex)
{
WulaLog.Debug($"[OmniRealtime] Failed to send text: {ex.Message}");
}
}
/// <summary>
/// 发送图像并请求分析
/// </summary>
public async Task SendImageAndRequestAnalysisAsync(string base64Image, string prompt = "分析当前游戏画面,决定下一步操作")
{
if (!IsConnected) return;
try
{
// 先发送图像
await SendImageAsync(base64Image);
// 提交输入并请求响应
var commitEvent = new
{
event_id = GenerateEventId(),
type = "input_audio_buffer.commit" // 这会同时提交图像缓冲区
};
await SendEventAsync(commitEvent);
// 发送文本提示
await SendTextAndRequestResponseAsync(prompt);
}
catch (Exception ex)
{
WulaLog.Debug($"[OmniRealtime] Failed to send image for analysis: {ex.Message}");
}
}
/// <summary>
/// 接收消息循环
/// </summary>
private async Task ReceiveLoopAsync()
{
var buffer = new byte[8192];
var messageBuffer = new StringBuilder();
try
{
while (_webSocket?.State == WebSocketState.Open && !_cancellationSource.IsCancellationRequested)
{
var segment = new ArraySegment<byte>(buffer);
var result = await _webSocket.ReceiveAsync(segment, _cancellationSource.Token);
if (result.MessageType == WebSocketMessageType.Close)
{
WulaLog.Debug("[OmniRealtime] Server closed connection");
break;
}
if (result.MessageType == WebSocketMessageType.Text)
{
string chunk = Encoding.UTF8.GetString(buffer, 0, result.Count);
messageBuffer.Append(chunk);
if (result.EndOfMessage)
{
ProcessServerEvent(messageBuffer.ToString());
messageBuffer.Clear();
}
}
}
}
catch (OperationCanceledException)
{
// 正常取消
}
catch (Exception ex)
{
WulaLog.Debug($"[OmniRealtime] Receive error: {ex.Message}");
OnError?.Invoke($"接收错误: {ex.Message}");
}
finally
{
_isConnected = false;
OnDisconnected?.Invoke();
}
}
/// <summary>
/// 处理服务端事件
/// </summary>
private void ProcessServerEvent(string json)
{
try
{
// 简单解析 JSON 获取事件类型和内容
// 注意:这里使用简单的字符串解析,生产环境应使用 JSON 库
string eventType = ExtractJsonValue(json, "type");
switch (eventType)
{
case "session.created":
case "session.updated":
WulaLog.Debug($"[OmniRealtime] Session event: {eventType}");
break;
case "response.text.delta":
string textDelta = ExtractJsonValue(json, "delta");
if (!string.IsNullOrEmpty(textDelta))
{
_currentResponse.Append(textDelta);
OnTextDelta?.Invoke(textDelta);
}
break;
case "response.text.done":
string completeText = _currentResponse.ToString();
_currentResponse.Clear();
OnTextComplete?.Invoke(completeText);
WulaLog.Debug($"[OmniRealtime] Response complete: {completeText.Substring(0, Math.Min(100, completeText.Length))}...");
break;
case "response.audio_transcript.delta":
// 音频转录的文本增量(如果启用了音频输出)
string transcriptDelta = ExtractJsonValue(json, "delta");
if (!string.IsNullOrEmpty(transcriptDelta))
{
_currentResponse.Append(transcriptDelta);
OnTextDelta?.Invoke(transcriptDelta);
}
break;
case "response.audio_transcript.done":
string transcript = _currentResponse.ToString();
_currentResponse.Clear();
OnTextComplete?.Invoke(transcript);
break;
case "error":
string errorMsg = ExtractJsonValue(json, "message") ?? json;
WulaLog.Debug($"[OmniRealtime] Error: {errorMsg}");
OnError?.Invoke(errorMsg);
break;
case "response.done":
// 响应完成
break;
default:
// 其他事件
if (Prefs.DevMode)
{
WulaLog.Debug($"[OmniRealtime] Event: {eventType}");
}
break;
}
}
catch (Exception ex)
{
WulaLog.Debug($"[OmniRealtime] Failed to process event: {ex.Message}");
}
}
private async Task SendEventAsync(object eventObj)
{
if (_webSocket?.State != WebSocketState.Open) return;
string json = ToSimpleJson(eventObj);
byte[] data = Encoding.UTF8.GetBytes(json);
await _webSocket.SendAsync(
new ArraySegment<byte>(data),
WebSocketMessageType.Text,
true,
_cancellationSource.Token
);
}
private static string GenerateEventId()
{
return $"event_{Guid.NewGuid():N}".Substring(0, 24);
}
/// <summary>
/// 简单的对象转 JSON避免依赖外部库
/// </summary>
private static string ToSimpleJson(object obj)
{
// 对于复杂对象,建议使用 Newtonsoft.Json 或 System.Text.Json
// 这里使用简化实现
var sb = new StringBuilder();
SerializeObject(sb, obj);
return sb.ToString();
}
private static void SerializeObject(StringBuilder sb, object obj)
{
if (obj == null)
{
sb.Append("null");
return;
}
var type = obj.GetType();
if (obj is string str)
{
sb.Append('"');
sb.Append(str.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\n", "\\n").Replace("\r", "\\r"));
sb.Append('"');
}
else if (obj is bool b)
{
sb.Append(b ? "true" : "false");
}
else if (obj is int || obj is long || obj is float || obj is double)
{
sb.Append(obj.ToString());
}
else if (type.IsArray)
{
sb.Append('[');
var arr = (Array)obj;
for (int i = 0; i < arr.Length; i++)
{
if (i > 0) sb.Append(',');
SerializeObject(sb, arr.GetValue(i));
}
sb.Append(']');
}
else if (type.IsClass)
{
sb.Append('{');
bool first = true;
foreach (var prop in type.GetProperties())
{
var value = prop.GetValue(obj);
if (value == null) continue;
if (!first) sb.Append(',');
first = false;
sb.Append('"');
sb.Append(prop.Name);
sb.Append("\":");
SerializeObject(sb, value);
}
// 匿名类型使用字段
foreach (var field in type.GetFields())
{
var value = field.GetValue(obj);
if (value == null) continue;
if (!first) sb.Append(',');
first = false;
sb.Append('"');
sb.Append(field.Name);
sb.Append("\":");
SerializeObject(sb, value);
}
sb.Append('}');
}
}
private static string ExtractJsonValue(string json, string key)
{
// 简单提取 JSON 值
string pattern = $"\"{key}\":";
int idx = json.IndexOf(pattern);
if (idx < 0) return null;
idx += pattern.Length;
while (idx < json.Length && char.IsWhiteSpace(json[idx])) idx++;
if (idx >= json.Length) return null;
if (json[idx] == '"')
{
// 字符串值
idx++;
int end = idx;
while (end < json.Length && json[end] != '"')
{
if (json[end] == '\\') end++; // 跳过转义字符
end++;
}
return json.Substring(idx, end - idx).Replace("\\n", "\n").Replace("\\\"", "\"");
}
else
{
// 其他值
int end = idx;
while (end < json.Length && json[end] != ',' && json[end] != '}' && json[end] != ']')
{
end++;
}
return json.Substring(idx, end - idx).Trim();
}
}
/// <summary>
/// 断开连接
/// </summary>
public async Task DisconnectAsync()
{
if (!_isConnected) return;
try
{
_cancellationSource?.Cancel();
if (_webSocket?.State == WebSocketState.Open)
{
await _webSocket.CloseAsync(WebSocketCloseStatus.NormalClosure, "Client disconnect", CancellationToken.None);
}
}
catch (Exception ex)
{
WulaLog.Debug($"[OmniRealtime] Disconnect error: {ex.Message}");
}
finally
{
_isConnected = false;
_webSocket?.Dispose();
_webSocket = null;
}
}
public void Dispose()
{
if (_isDisposed) return;
_isDisposed = true;
_cancellationSource?.Cancel();
_cancellationSource?.Dispose();
_webSocket?.Dispose();
}
}
}

View File

@@ -0,0 +1,285 @@
using System;
using System.Collections.Generic;
using System.Linq;
using RimWorld;
using RimWorld.Planet;
using Verse;
using Verse.AI;
namespace WulaFallenEmpire.EventSystem.AI.Agent
{
/// <summary>
/// 游戏状态观察器 - 收集当前游戏状态用于 AI 决策
/// </summary>
public static class StateObserver
{
/// <summary>
/// 捕获当前游戏状态快照
/// </summary>
public static GameStateSnapshot CaptureState()
{
var snapshot = new GameStateSnapshot();
try
{
Map map = Find.CurrentMap ?? Find.Maps?.FirstOrDefault();
if (map == null)
{
WulaLog.Debug("[StateObserver] No map available");
return snapshot;
}
// 时间信息
CaptureTimeInfo(snapshot);
// 环境信息
CaptureEnvironmentInfo(snapshot, map);
// 殖民者信息
CaptureColonistInfo(snapshot, map);
// 资源统计
CaptureResourceInfo(snapshot, map);
// 建筑信息
CaptureBuildingInfo(snapshot, map);
// 威胁检测
CaptureThreatInfo(snapshot, map);
// 最近消息
CaptureRecentMessages(snapshot);
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing state: {ex.Message}");
}
return snapshot;
}
private static void CaptureTimeInfo(GameStateSnapshot snapshot)
{
try
{
var tickManager = Find.TickManager;
if (tickManager == null) return;
int ticksAbs = Find.TickManager.TicksAbs;
int tile = Find.CurrentMap?.Tile ?? 0;
float longitude = Find.WorldGrid.LongLatOf(tile).x;
snapshot.Hour = GenDate.HourOfDay(ticksAbs, longitude);
snapshot.DayOfQuadrum = GenDate.DayOfQuadrum(ticksAbs, longitude) + 1;
snapshot.Year = GenDate.Year(ticksAbs, longitude);
snapshot.Season = GenDate.Season(ticksAbs, Find.WorldGrid.LongLatOf(tile)).Label();
snapshot.GameSpeedMultiplier = Find.TickManager.TickRateMultiplier;
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing time: {ex.Message}");
}
}
private static void CaptureEnvironmentInfo(GameStateSnapshot snapshot, Map map)
{
try
{
snapshot.BiomeName = map.Biome?.label ?? "Unknown";
snapshot.OutdoorTemperature = map.mapTemperature?.OutdoorTemp ?? 20f;
snapshot.Weather = map.weatherManager?.curWeather?.label ?? "Unknown";
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing environment: {ex.Message}");
}
}
private static void CaptureColonistInfo(GameStateSnapshot snapshot, Map map)
{
try
{
var colonists = map.mapPawns?.FreeColonists;
if (colonists == null) return;
foreach (var pawn in colonists)
{
if (pawn == null || pawn.Dead) continue;
var pawnSnapshot = new PawnSnapshot
{
Name = pawn.LabelShortCap,
HealthPercent = pawn.health?.summaryHealth?.SummaryHealthPercent ?? 1f,
MoodPercent = pawn.needs?.mood?.CurLevelPercentage ?? 0.5f,
IsDrafted = pawn.Drafted,
IsDowned = pawn.Downed,
CurrentJob = GetJobDescription(pawn),
X = pawn.Position.x,
Z = pawn.Position.z
};
snapshot.Colonists.Add(pawnSnapshot);
}
// 囚犯
var prisoners = map.mapPawns?.PrisonersOfColony;
if (prisoners != null)
{
foreach (var pawn in prisoners)
{
if (pawn == null || pawn.Dead) continue;
snapshot.Prisoners.Add(new PawnSnapshot
{
Name = pawn.LabelShortCap,
HealthPercent = pawn.health?.summaryHealth?.SummaryHealthPercent ?? 1f,
IsDowned = pawn.Downed
});
}
}
// 驯养动物
var animals = map.mapPawns?.SpawnedColonyAnimals;
if (animals != null)
{
snapshot.Animals.AddRange(animals.Take(20).Select(a => new PawnSnapshot
{
Name = a.LabelShortCap,
HealthPercent = a.health?.summaryHealth?.SummaryHealthPercent ?? 1f
}));
}
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing colonists: {ex.Message}");
}
}
private static string GetJobDescription(Pawn pawn)
{
try
{
if (pawn.Drafted) return "已征召";
if (pawn.Downed) return "倒地";
if (pawn.InMentalState) return $"精神崩溃: {pawn.MentalStateDef?.label ?? ""}";
var job = pawn.CurJob;
if (job == null) return "空闲";
// 返回简短的工作描述
string jobLabel = job.def?.reportString ?? job.def?.label ?? "工作中";
// 清理一些常见的格式
if (job.targetA.Thing != null)
{
jobLabel = $"{job.def?.label}: {job.targetA.Thing.LabelShortCap}";
}
else
{
jobLabel = job.def?.label ?? "工作中";
}
return jobLabel;
}
catch
{
return "未知";
}
}
private static void CaptureResourceInfo(GameStateSnapshot snapshot, Map map)
{
try
{
var resourceCounter = map.resourceCounter;
if (resourceCounter == null) return;
foreach (var kvp in resourceCounter.AllCountedAmounts)
{
if (kvp.Value > 0)
{
snapshot.Resources[kvp.Key.defName] = kvp.Value;
}
}
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing resources: {ex.Message}");
}
}
private static void CaptureBuildingInfo(GameStateSnapshot snapshot, Map map)
{
try
{
// 统计建筑
snapshot.TotalBuildings = map.listerBuildings?.allBuildingsColonist?.Count ?? 0;
// 统计蓝图
snapshot.PendingBlueprints = map.listerThings?.ThingsInGroup(ThingRequestGroup.Blueprint)?.Count ?? 0;
// 统计建造中的框架
snapshot.ConstructionFrames = map.listerThings?.ThingsInGroup(ThingRequestGroup.BuildingFrame)?.Count ?? 0;
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing buildings: {ex.Message}");
}
}
private static void CaptureThreatInfo(GameStateSnapshot snapshot, Map map)
{
try
{
// 检测敌对 pawns
var hostilePawns = map.mapPawns?.AllPawnsSpawned?
.Where(p => p != null && !p.Dead && p.HostileTo(Faction.OfPlayer))
.ToList();
if (hostilePawns == null || hostilePawns.Count == 0) return;
// 按派系分组
var threatGroups = hostilePawns.GroupBy(p => p.Faction?.Name ?? "野生动物");
foreach (var group in threatGroups)
{
IntVec3 colonyCenter = map.IsPlayerHome ? map.mapPawns.FreeColonists.FirstOrDefault()?.Position ?? IntVec3.Zero : IntVec3.Zero;
float minDistance = group.Min(p => (p.Position - colonyCenter).LengthHorizontal);
snapshot.Threats.Add(new ThreatSnapshot
{
Description = $"{group.Count()} 个 {group.Key}",
Count = group.Count(),
Distance = minDistance,
IsHostile = true
});
}
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing threats: {ex.Message}");
}
}
private static void CaptureRecentMessages(GameStateSnapshot snapshot)
{
try
{
var messages = Find.Archive?.ArchivablesListForReading;
if (messages == null) return;
// 获取最近 5 条消息
var allMessages = messages.OfType<Message>().ToList();
var recentMessages = allMessages
.Skip(Math.Max(0, allMessages.Count - 5))
.Select(m => m.text?.ToString() ?? "")
.Where(s => !string.IsNullOrEmpty(s))
.ToList();
snapshot.RecentMessages.AddRange(recentMessages);
}
catch (Exception ex)
{
WulaLog.Debug($"[StateObserver] Error capturing messages: {ex.Message}");
}
}
}
}

View File

@@ -0,0 +1,139 @@
using System;
using System.Collections.Generic;
using System.Linq;
using RimWorld;
using Verse;
namespace WulaFallenEmpire.EventSystem.AI.Tools
{
/// <summary>
/// 采矿指令工具 - 在指定位置或区域添加采矿标记
/// </summary>
public class Tool_DesignateMine : AITool
{
public override string Name => "designate_mine";
public override string Description =>
"在指定坐标添加采矿标记。可以指定单个格子或矩形区域。只能标记可采矿的岩石。";
public override string UsageSchema =>
"<designate_mine><x>整数X坐标</x><z>整数Z坐标</z><radius>可选整数半径默认0表示单格</radius></designate_mine>";
public override string Execute(string args)
{
try
{
var argsDict = ParseXmlArgs(args);
// 解析坐标
if (!argsDict.TryGetValue("x", out string xStr) || !int.TryParse(xStr, out int x))
{
return "Error: 缺少有效的 x 坐标";
}
if (!argsDict.TryGetValue("z", out string zStr) || !int.TryParse(zStr, out int z))
{
return "Error: 缺少有效的 z 坐标";
}
int radius = 0;
if (argsDict.TryGetValue("radius", out string radiusStr))
{
int.TryParse(radiusStr, out radius);
}
radius = Math.Max(0, Math.Min(10, radius)); // 限制半径 0-10
// 获取地图
Map map = Find.CurrentMap;
if (map == null)
{
return "Error: 没有活动的地图";
}
IntVec3 center = new IntVec3(x, 0, z);
if (!center.InBounds(map))
{
return $"Error: 坐标 ({x}, {z}) 超出地图边界";
}
// 收集要标记的格子
List<IntVec3> cellsToMark = new List<IntVec3>();
if (radius == 0)
{
cellsToMark.Add(center);
}
else
{
// 矩形区域
for (int dx = -radius; dx <= radius; dx++)
{
for (int dz = -radius; dz <= radius; dz++)
{
IntVec3 cell = new IntVec3(x + dx, 0, z + dz);
if (cell.InBounds(map))
{
cellsToMark.Add(cell);
}
}
}
}
int successCount = 0;
int alreadyMarkedCount = 0;
int notMineableCount = 0;
foreach (var cell in cellsToMark)
{
// 检查是否已有采矿标记
if (map.designationManager.DesignationAt(cell, DesignationDefOf.Mine) != null)
{
alreadyMarkedCount++;
continue;
}
// 检查是否可采矿
Mineable mineable = cell.GetFirstMineable(map);
if (mineable == null)
{
notMineableCount++;
continue;
}
// 添加采矿标记
map.designationManager.AddDesignation(new Designation(cell, DesignationDefOf.Mine));
successCount++;
}
// 生成结果报告
if (successCount > 0)
{
string result = $"Success: 已标记 {successCount} 个格子进行采矿";
if (alreadyMarkedCount > 0)
{
result += $"{alreadyMarkedCount} 个已有标记";
}
if (notMineableCount > 0)
{
result += $"{notMineableCount} 个不可采矿";
}
Messages.Message($"AI: 标记了 {successCount} 处采矿", MessageTypeDefOf.NeutralEvent);
return result;
}
else if (alreadyMarkedCount > 0)
{
return $"Info: 该区域 {alreadyMarkedCount} 个格子已有采矿标记";
}
else
{
return $"Error: 坐标 ({x}, {z}) 附近没有可采矿的岩石";
}
}
catch (Exception ex)
{
WulaLog.Debug($"[Tool_DesignateMine] Error: {ex}");
return $"Error: 采矿指令失败 - {ex.Message}";
}
}
}
}

View File

@@ -0,0 +1,105 @@
using System;
using RimWorld;
using Verse;
namespace WulaFallenEmpire.EventSystem.AI.Tools
{
/// <summary>
/// 征召殖民者工具 - 将殖民者置于征召状态以便直接控制
/// </summary>
public class Tool_DraftPawn : AITool
{
public override string Name => "draft_pawn";
public override string Description =>
"征召或解除征召殖民者。征召后可以直接控制殖民者移动和攻击。";
public override string UsageSchema =>
"<draft_pawn><pawn_name>殖民者名字</pawn_name><draft>true征召/false解除</draft></draft_pawn>";
public override string Execute(string args)
{
try
{
var argsDict = ParseXmlArgs(args);
// 解析殖民者名字
if (!argsDict.TryGetValue("pawn_name", out string pawnName) || string.IsNullOrWhiteSpace(pawnName))
{
// 尝试其他常见参数名
if (!argsDict.TryGetValue("name", out pawnName) || string.IsNullOrWhiteSpace(pawnName))
{
return "Error: 缺少殖民者名字 (pawn_name)";
}
}
// 解析征召状态
bool draft = true;
if (argsDict.TryGetValue("draft", out string draftStr))
{
draft = draftStr.ToLowerInvariant() != "false" && draftStr != "0";
}
// 获取地图
Map map = Find.CurrentMap;
if (map == null)
{
return "Error: 没有活动的地图";
}
// 查找殖民者
Pawn targetPawn = null;
foreach (var pawn in map.mapPawns.FreeColonists)
{
if (pawn.LabelShortCap.Equals(pawnName, StringComparison.OrdinalIgnoreCase) ||
pawn.Name?.ToStringShort?.Equals(pawnName, StringComparison.OrdinalIgnoreCase) == true ||
pawn.LabelCap.ToString().IndexOf(pawnName, StringComparison.OrdinalIgnoreCase) >= 0)
{
targetPawn = pawn;
break;
}
}
if (targetPawn == null)
{
return $"Error: 找不到殖民者 '{pawnName}'";
}
// 检查是否可以征召
if (targetPawn.Downed)
{
return $"Error: {targetPawn.LabelShortCap} 已倒地,无法征召";
}
if (targetPawn.Dead)
{
return $"Error: {targetPawn.LabelShortCap} 已死亡";
}
if (targetPawn.drafter == null)
{
return $"Error: {targetPawn.LabelShortCap} 无法被征召";
}
// 执行征召/解除
bool wasDrafted = targetPawn.Drafted;
targetPawn.drafter.Drafted = draft;
string action = draft ? "征召" : "解除征召";
if (wasDrafted == draft)
{
return $"Info: {targetPawn.LabelShortCap} 已经处于{(draft ? "" : "")}状态";
}
Messages.Message($"AI: {action}了 {targetPawn.LabelShortCap}", targetPawn, MessageTypeDefOf.NeutralEvent);
return $"Success: 已{action} {targetPawn.LabelShortCap},当前位置 ({targetPawn.Position.x}, {targetPawn.Position.z})";
}
catch (Exception ex)
{
WulaLog.Debug($"[Tool_DraftPawn] Error: {ex}");
return $"Error: 征召操作失败 - {ex.Message}";
}
}
}
}

View File

@@ -0,0 +1,45 @@
using System;
namespace WulaFallenEmpire.EventSystem.AI.Tools
{
/// <summary>
/// 获取当前游戏状态工具 - 让 AI 了解殖民地当前情况
/// </summary>
public class Tool_GetGameState : AITool
{
public override string Name => "get_game_state";
public override string Description =>
"获取当前游戏状态的详细报告,包括殖民者状态、资源、建筑进度、威胁等信息。在做出任何操作决策前应先调用此工具了解当前情况。";
public override string UsageSchema =>
"<get_game_state/>";
public override string Execute(string args)
{
try
{
var snapshot = Agent.StateObserver.CaptureState();
if (snapshot == null)
{
return "Error: 无法捕获游戏状态,可能没有活动的地图。";
}
string stateText = snapshot.ToPromptText();
if (string.IsNullOrWhiteSpace(stateText))
{
return "Error: 游戏状态为空。";
}
return stateText;
}
catch (Exception ex)
{
WulaLog.Debug($"[Tool_GetGameState] Error: {ex}");
return $"Error: 获取游戏状态失败 - {ex.Message}";
}
}
}
}

View File

@@ -0,0 +1,166 @@
using System;
using System.Threading.Tasks;
using UnityEngine;
namespace WulaFallenEmpire.EventSystem.AI.Tools
{
/// <summary>
/// 视觉点击工具 - 使用 VLM 分析屏幕后模拟鼠标点击
/// 适用于原版 API 无法直接操作的 mod UI 元素
/// </summary>
public class Tool_VisualClick : AITool
{
public override string Name => "visual_click";
public override string Description =>
"在指定的屏幕位置执行鼠标点击。坐标使用比例值 (0-1)(0,0) 是左上角,(1,1) 是右下角。" +
"适用于点击无法通过 API 操作的 mod 按钮或 UI 元素。先使用 analyze_screen 获取目标位置。";
public override string UsageSchema =>
"<visual_click><x>0-1之间的X比例</x><y>0-1之间的Y比例</y><right_click>可选true为右键</right_click></visual_click>";
public override string Execute(string args)
{
try
{
var argsDict = ParseXmlArgs(args);
// 解析 X 坐标
if (!argsDict.TryGetValue("x", out string xStr) || !float.TryParse(xStr, out float x))
{
return "Error: 缺少有效的 x 坐标 (0-1之间的比例值)";
}
// 解析 Y 坐标
if (!argsDict.TryGetValue("y", out string yStr) || !float.TryParse(yStr, out float y))
{
return "Error: 缺少有效的 y 坐标 (0-1之间的比例值)";
}
// 验证范围
if (x < 0 || x > 1 || y < 0 || y > 1)
{
return $"Error: 坐标 ({x}, {y}) 超出范围,必须在 0-1 之间";
}
// 解析右键选项
bool rightClick = false;
if (argsDict.TryGetValue("right_click", out string rightStr))
{
rightClick = rightStr.ToLowerInvariant() == "true" || rightStr == "1";
}
// 执行点击
bool success = Agent.MouseSimulator.ClickAtProportional(x, y, rightClick);
if (success)
{
string clickType = rightClick ? "右键" : "左键";
int screenX = Mathf.RoundToInt(x * Screen.width);
int screenY = Mathf.RoundToInt(y * Screen.height);
WulaLog.Debug($"[Tool_VisualClick] {clickType}点击 ({x:F3}, {y:F3}) -> 屏幕 ({screenX}, {screenY})");
return $"Success: 已在屏幕位置 ({screenX}, {screenY}) 执行{clickType}点击";
}
else
{
return "Error: 点击操作失败";
}
}
catch (Exception ex)
{
WulaLog.Debug($"[Tool_VisualClick] Error: {ex}");
return $"Error: 点击操作失败 - {ex.Message}";
}
}
}
/// <summary>
/// 视觉输入文本工具 - 在当前焦点位置输入文本
/// </summary>
public class Tool_VisualTypeText : AITool
{
public override string Name => "visual_type_text";
public override string Description =>
"在当前焦点位置输入文本。适用于需要文本输入的对话框或输入框。应先用 visual_click 点击输入框获取焦点。";
public override string UsageSchema =>
"<visual_type_text><text>要输入的文本</text></visual_type_text>";
public override string Execute(string args)
{
try
{
var argsDict = ParseXmlArgs(args);
if (!argsDict.TryGetValue("text", out string text) || string.IsNullOrEmpty(text))
{
return "Error: 缺少要输入的文本";
}
// 使用剪贴板方式输入(支持中文)
GUIUtility.systemCopyBuffer = text;
// 模拟 Ctrl+V 粘贴
// 注意:这需要额外的键盘模拟实现
// 暂时返回成功,实际使用时需要完善
WulaLog.Debug($"[Tool_VisualTypeText] 已将文本复制到剪贴板: {text}");
return $"Success: 已将文本复制到剪贴板。请手动按 Ctrl+V 粘贴,或等待键盘模拟功能完善。";
}
catch (Exception ex)
{
WulaLog.Debug($"[Tool_VisualTypeText] Error: {ex}");
return $"Error: 输入文本失败 - {ex.Message}";
}
}
}
/// <summary>
/// 视觉滚动工具 - 在当前位置滚动鼠标滚轮
/// </summary>
public class Tool_VisualScroll : AITool
{
public override string Name => "visual_scroll";
public override string Description =>
"在当前鼠标位置滚动。可选先移动到指定位置再滚动。delta 正数向上滚动,负数向下滚动。";
public override string UsageSchema =>
"<visual_scroll><delta>滚动量,正数向上负数向下</delta><x>可选0-1 X坐标</x><y>可选0-1 Y坐标</y></visual_scroll>";
public override string Execute(string args)
{
try
{
var argsDict = ParseXmlArgs(args);
if (!argsDict.TryGetValue("delta", out string deltaStr) || !int.TryParse(deltaStr, out int delta))
{
return "Error: 缺少有效的 delta 值";
}
// 可选:先移动到指定位置
if (argsDict.TryGetValue("x", out string xStr) && argsDict.TryGetValue("y", out string yStr))
{
if (float.TryParse(xStr, out float x) && float.TryParse(yStr, out float y))
{
Agent.MouseSimulator.MoveToProportional(x, y);
System.Threading.Thread.Sleep(10);
}
}
Agent.MouseSimulator.Scroll(delta);
string direction = delta > 0 ? "向上" : "向下";
return $"Success: 已{direction}滚动 {Math.Abs(delta)} 单位";
}
catch (Exception ex)
{
WulaLog.Debug($"[Tool_VisualScroll] Error: {ex}");
return $"Error: 滚动操作失败 - {ex.Message}";
}
}
}
}