diff --git a/1.6/1.6/Assemblies/WulaFallenEmpire.dll b/1.6/1.6/Assemblies/WulaFallenEmpire.dll index 476dd34e..2189bb08 100644 Binary files a/1.6/1.6/Assemblies/WulaFallenEmpire.dll and b/1.6/1.6/Assemblies/WulaFallenEmpire.dll differ diff --git a/1.6/1.6/Assemblies/WulaFallenEmpire.pdb b/1.6/1.6/Assemblies/WulaFallenEmpire.pdb index dfd5ff62..77032d12 100644 Binary files a/1.6/1.6/Assemblies/WulaFallenEmpire.pdb and b/1.6/1.6/Assemblies/WulaFallenEmpire.pdb differ diff --git a/Source/WulaFallenEmpire/EventSystem/AI/Agent/AutonomousAgentLoop.cs b/Source/WulaFallenEmpire/EventSystem/AI/Agent/AutonomousAgentLoop.cs new file mode 100644 index 00000000..454e5410 --- /dev/null +++ b/Source/WulaFallenEmpire/EventSystem/AI/Agent/AutonomousAgentLoop.cs @@ -0,0 +1,278 @@ +using System; +using System.Collections.Generic; +using System.Text; +using System.Threading.Tasks; +using RimWorld; +using UnityEngine; +using Verse; + +namespace WulaFallenEmpire.EventSystem.AI.Agent +{ + /// + /// 自主 Agent 循环 - 持续观察游戏并做出决策 + /// 用户只需给出开放式指令如"帮我挖铁"或"帮我玩殖民地" + /// + public class AutonomousAgentLoop : GameComponent + { + public static AutonomousAgentLoop Instance { get; private set; } + + // Agent 状态 + private bool _isRunning; + private string _currentObjective; + private float _lastDecisionTime; + private int _decisionCount; + private readonly List _actionHistory = new List(); + + // 配置 + private const float DecisionIntervalSeconds = 3f; // 每 3 秒决策一次 + private const int MaxActionsPerObjective = 100; + + // 事件 + public event Action OnDecisionMade; + public event Action OnObjectiveComplete; + public event Action OnError; + + public bool IsRunning => _isRunning; + public string CurrentObjective => _currentObjective; + public int DecisionCount => _decisionCount; + + public AutonomousAgentLoop(Game game) + { + Instance = this; + } + + /// + /// 开始执行开放式目标 + /// + public void StartObjective(string objective) + { + if (string.IsNullOrWhiteSpace(objective)) + { + OnError?.Invoke("目标不能为空"); + return; + } + + _currentObjective = objective; + _isRunning = true; + _decisionCount = 0; + _actionHistory.Clear(); + _lastDecisionTime = Time.realtimeSinceStartup; + + WulaLog.Debug($"[AgentLoop] Started objective: {objective}"); + Messages.Message($"AI Agent 开始执行: {objective}", MessageTypeDefOf.NeutralEvent); + + // 立即执行第一次决策 + _ = ExecuteDecisionCycleAsync(); + } + + /// + /// 停止 Agent + /// + public void Stop() + { + if (!_isRunning) return; + + _isRunning = false; + WulaLog.Debug($"[AgentLoop] Stopped after {_decisionCount} decisions"); + Messages.Message($"AI Agent 已停止,执行了 {_decisionCount} 次决策", MessageTypeDefOf.NeutralEvent); + } + + public override void GameComponentTick() + { + if (!_isRunning) return; + + // 检查是否到达决策间隔 + if (Time.realtimeSinceStartup - _lastDecisionTime < DecisionIntervalSeconds) return; + + // 检查是否超过最大操作次数 + if (_decisionCount >= MaxActionsPerObjective) + { + Messages.Message($"AI Agent: 已达到最大操作次数 ({MaxActionsPerObjective}),暂停执行", MessageTypeDefOf.CautionInput); + Stop(); + return; + } + + _lastDecisionTime = Time.realtimeSinceStartup; + + // 异步执行决策 + _ = ExecuteDecisionCycleAsync(); + } + + /// + /// 执行一次决策循环: Observe → Think → Act + /// + private async Task ExecuteDecisionCycleAsync() + { + try + { + // 1. Observe - 收集游戏状态 + var gameState = StateObserver.CaptureState(); + string stateText = gameState.ToPromptText(); + + // 2. 构建决策提示词 + string prompt = BuildDecisionPrompt(stateText); + + // 3. Think - 调用 AI 获取决策 + var settings = WulaFallenEmpireMod.settings; + if (settings == null || string.IsNullOrEmpty(settings.apiKey)) + { + OnError?.Invoke("API Key 未配置"); + Stop(); + return; + } + + var client = new SimpleAIClient(settings.apiKey, settings.baseUrl, settings.model); + + // 使用 VLM 如果启用且配置了 + string decision; + if (settings.enableVlmFeatures && !string.IsNullOrEmpty(settings.vlmApiKey)) + { + // 使用 VLM 分析屏幕 + string base64Image = ScreenCaptureUtility.CaptureScreenAsBase64(); + var vlmClient = new SimpleAIClient(settings.vlmApiKey, settings.vlmBaseUrl, settings.vlmModel); + decision = await vlmClient.GetVisionCompletionAsync( + GetAgentSystemPrompt(), + prompt, + base64Image, + maxTokens: 512, + temperature: 0.3f + ); + } + else + { + // 纯文本模式 + var messages = new List<(string role, string message)> + { + ("user", prompt) + }; + decision = await client.GetChatCompletionAsync(GetAgentSystemPrompt(), messages, 512, 0.3f); + } + + if (string.IsNullOrEmpty(decision)) + { + WulaLog.Debug("[AgentLoop] Empty decision received"); + return; + } + + _decisionCount++; + WulaLog.Debug($"[AgentLoop] Decision #{_decisionCount}: {decision.Substring(0, Math.Min(100, decision.Length))}..."); + + // 4. Act - 执行决策 + ExecuteDecision(decision); + + // 5. 记录历史 + _actionHistory.Add($"[{_decisionCount}] {decision.Substring(0, Math.Min(50, decision.Length))}"); + if (_actionHistory.Count > 20) + { + _actionHistory.RemoveAt(0); + } + + OnDecisionMade?.Invoke(decision); + + // 6. 检查是否完成目标 + if (decision.Contains(" 0) + { + sb.AppendLine(); + sb.AppendLine("# 最近操作历史"); + foreach (var action in _actionHistory) + { + sb.AppendLine($"- {action}"); + } + } + + sb.AppendLine(); + sb.AppendLine("# 请决定下一步操作"); + sb.AppendLine("分析当前状态,输出一个 XML 工具调用来推进目标。"); + sb.AppendLine("如果目标已完成,输出 。"); + sb.AppendLine("如果不需要操作(等待中),输出 。"); + + return sb.ToString(); + } + + private string GetAgentSystemPrompt() + { + return @"你是一个自主 RimWorld 游戏 AI Agent。你的任务是独立完成用户给出的开放式目标。 + +# 核心原则 +1. **自主决策**: 不要等待用户指示,主动分析情况并采取行动 +2. **循序渐进**: 每次只执行一个操作,观察结果后再决定下一步 +3. **问题应对**: 遇到障碍时自己想办法解决 +4. **目标导向**: 始终围绕目标推进,避免无关操作 + +# 可用工具 +- get_game_state: 获取详细游戏状态 +- designate_mine: X坐标Z坐标可选半径 标记采矿 +- draft_pawn: 名字true/false 征召殖民者 +- analyze_screen: 分析目标 分析屏幕(需要VLM) +- visual_click: 0-1比例0-1比例 模拟点击 + +# 输出格式 +直接输出一个 XML 工具调用,不要解释。 +如果目标已完成: +如果需要等待: + +# 注意事项 +- 坐标使用游戏内整数坐标,不是屏幕比例 +- 优先使用 API 工具(designate_mine 等),视觉工具用于 mod 内容 +- 保持简洁高效"; + } + + private void ExecuteDecision(string decision) + { + // 解析并执行 AI 的决策 + // 从 AIIntelligenceCore 借用工具执行逻辑 + + var core = AIIntelligenceCore.Instance; + if (core == null) + { + WulaLog.Debug("[AgentLoop] AIIntelligenceCore not available"); + return; + } + + // 提取工具调用并执行 + // 暂时使用简单的正则匹配,实际应整合 AIIntelligenceCore 的解析逻辑 + + if (decision.Contains(" + /// 纯视觉交互工具集 - 仿照 Python VLM Agent + /// 当没有原生 API 可用时,AI 可以通过这些工具操作任何界面 + /// + public static class VisualInteractionTools + { + // Windows API + [DllImport("user32.dll")] + private static extern bool SetCursorPos(int X, int Y); + + [DllImport("user32.dll")] + private static extern void mouse_event(uint dwFlags, uint dx, uint dy, uint dwData, int dwExtraInfo); + + [DllImport("user32.dll")] + private static extern void keybd_event(byte bVk, byte bScan, uint dwFlags, int dwExtraInfo); + + [DllImport("user32.dll")] + private static extern short VkKeyScan(char ch); + + // 鼠标事件标志 + private const uint MOUSEEVENTF_LEFTDOWN = 0x0002; + private const uint MOUSEEVENTF_LEFTUP = 0x0004; + private const uint MOUSEEVENTF_RIGHTDOWN = 0x0008; + private const uint MOUSEEVENTF_RIGHTUP = 0x0010; + private const uint MOUSEEVENTF_WHEEL = 0x0800; + + // 键盘事件标志 + private const uint KEYEVENTF_KEYDOWN = 0x0000; + private const uint KEYEVENTF_KEYUP = 0x0002; + + // 虚拟键码 + private const byte VK_CONTROL = 0x11; + private const byte VK_SHIFT = 0x10; + private const byte VK_ALT = 0x12; + private const byte VK_RETURN = 0x0D; + private const byte VK_BACK = 0x08; + private const byte VK_ESCAPE = 0x1B; + private const byte VK_TAB = 0x09; + private const byte VK_LWIN = 0x5B; + private const byte VK_F4 = 0x73; + + /// + /// 1. 鼠标点击 - 在比例坐标处点击 + /// + public static string MouseClick(float x, float y, string button = "left", int clicks = 1) + { + try + { + int screenX = Mathf.RoundToInt(x * Screen.width); + int screenY = Mathf.RoundToInt(y * Screen.height); + + // Unity Y 轴翻转 + int windowsY = Screen.height - screenY; + + SetCursorPos(screenX, windowsY); + Thread.Sleep(20); + + for (int i = 0; i < clicks; i++) + { + if (button == "right") + { + mouse_event(MOUSEEVENTF_RIGHTDOWN, 0, 0, 0, 0); + mouse_event(MOUSEEVENTF_RIGHTUP, 0, 0, 0, 0); + } + else + { + mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0); + mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0); + } + if (i < clicks - 1) Thread.Sleep(50); + } + + string buttonText = button == "right" ? "右键" : "左键"; + string clickText = clicks == 2 ? "双击" : "单击"; + return $"Success: 在 ({screenX}, {screenY}) 处{buttonText}{clickText}"; + } + catch (Exception ex) + { + return $"Error: 点击失败 - {ex.Message}"; + } + } + + /// + /// 2. 输入文本 - 在指定位置点击后输入文本(通过剪贴板) + /// + public static string TypeText(float x, float y, string text) + { + try + { + // 先点击 + MouseClick(x, y); + Thread.Sleep(100); + + // 通过剪贴板输入 + GUIUtility.systemCopyBuffer = text; + Thread.Sleep(50); + + // Ctrl+V 粘贴 + keybd_event(VK_CONTROL, 0, KEYEVENTF_KEYDOWN, 0); + keybd_event(0x56, 0, KEYEVENTF_KEYDOWN, 0); // V + keybd_event(0x56, 0, KEYEVENTF_KEYUP, 0); + keybd_event(VK_CONTROL, 0, KEYEVENTF_KEYUP, 0); + + return $"Success: 在 ({x:F3}, {y:F3}) 处输入文本: {text}"; + } + catch (Exception ex) + { + return $"Error: 输入文本失败 - {ex.Message}"; + } + } + + /// + /// 3. 滚动窗口 - 在指定位置滚动 + /// + public static string ScrollWindow(float x, float y, string direction = "up", int amount = 3) + { + try + { + int screenX = Mathf.RoundToInt(x * Screen.width); + int screenY = Mathf.RoundToInt(y * Screen.height); + int windowsY = Screen.height - screenY; + + SetCursorPos(screenX, windowsY); + Thread.Sleep(20); + + int wheelDelta = (direction == "up" ? 1 : -1) * 120 * amount; + mouse_event(MOUSEEVENTF_WHEEL, 0, 0, (uint)wheelDelta, 0); + + string dir = direction == "up" ? "向上" : "向下"; + return $"Success: 在 ({screenX}, {screenY}) 处{dir}滚动 {amount} 步"; + } + catch (Exception ex) + { + return $"Error: 滚动失败 - {ex.Message}"; + } + } + + /// + /// 4. 鼠标拖拽 - 从起点拖到终点 + /// + public static string MouseDrag(float startX, float startY, float endX, float endY, float durationSec = 0.5f) + { + try + { + int sx = Mathf.RoundToInt(startX * Screen.width); + int sy = Screen.height - Mathf.RoundToInt(startY * Screen.height); + int ex = Mathf.RoundToInt(endX * Screen.width); + int ey = Screen.height - Mathf.RoundToInt(endY * Screen.height); + + // 移动到起点 + SetCursorPos(sx, sy); + Thread.Sleep(50); + + // 按下 + mouse_event(MOUSEEVENTF_LEFTDOWN, 0, 0, 0, 0); + + // 平滑移动 + int steps = Mathf.Max(5, Mathf.RoundToInt(durationSec * 20)); + int delayMs = Mathf.RoundToInt(durationSec * 1000 / steps); + + for (int i = 1; i <= steps; i++) + { + float t = (float)i / steps; + int cx = Mathf.RoundToInt(Mathf.Lerp(sx, ex, t)); + int cy = Mathf.RoundToInt(Mathf.Lerp(sy, ey, t)); + SetCursorPos(cx, cy); + Thread.Sleep(delayMs); + } + + // 释放 + mouse_event(MOUSEEVENTF_LEFTUP, 0, 0, 0, 0); + + return $"Success: 从 ({startX:F3}, {startY:F3}) 拖拽到 ({endX:F3}, {endY:F3})"; + } + catch (Exception ex) + { + return $"Error: 拖拽失败 - {ex.Message}"; + } + } + + /// + /// 5. 等待 - 暂停指定秒数 + /// + public static string Wait(float seconds) + { + try + { + Thread.Sleep(Mathf.RoundToInt(seconds * 1000)); + return $"Success: 等待了 {seconds} 秒"; + } + catch (Exception ex) + { + return $"Error: 等待失败 - {ex.Message}"; + } + } + + /// + /// 6. 按下回车键 + /// + public static string PressEnter() + { + try + { + keybd_event(VK_RETURN, 0, KEYEVENTF_KEYDOWN, 0); + keybd_event(VK_RETURN, 0, KEYEVENTF_KEYUP, 0); + return "Success: 按下回车键"; + } + catch (Exception ex) + { + return $"Error: 按键失败 - {ex.Message}"; + } + } + + /// + /// 7. 按下 Escape 键 + /// + public static string PressEscape() + { + try + { + keybd_event(VK_ESCAPE, 0, KEYEVENTF_KEYDOWN, 0); + keybd_event(VK_ESCAPE, 0, KEYEVENTF_KEYUP, 0); + return "Success: 按下 Escape 键"; + } + catch (Exception ex) + { + return $"Error: 按键失败 - {ex.Message}"; + } + } + + /// + /// 8. 删除文本 - 按 Backspace 删除指定数量字符 + /// + public static string DeleteText(float x, float y, int count = 1) + { + try + { + MouseClick(x, y); + Thread.Sleep(100); + + for (int i = 0; i < count; i++) + { + keybd_event(VK_BACK, 0, KEYEVENTF_KEYDOWN, 0); + keybd_event(VK_BACK, 0, KEYEVENTF_KEYUP, 0); + Thread.Sleep(20); + } + + return $"Success: 删除了 {count} 个字符"; + } + catch (Exception ex) + { + return $"Error: 删除失败 - {ex.Message}"; + } + } + + /// + /// 9. 执行快捷键 - 如 Ctrl+C, Alt+F4 等 + /// + public static string PressHotkey(float x, float y, string hotkey) + { + try + { + // 先点击获取焦点 + MouseClick(x, y); + Thread.Sleep(100); + + // 解析快捷键 + var keys = hotkey.ToLowerInvariant().Replace("+", " ").Replace("-", " ").Split(new[] { ' ' }, StringSplitOptions.RemoveEmptyEntries); + + // 按下修饰键 + foreach (var key in keys) + { + byte vk = GetVirtualKeyCode(key); + if (vk != 0) + { + keybd_event(vk, 0, KEYEVENTF_KEYDOWN, 0); + } + } + + Thread.Sleep(50); + + // 释放修饰键(逆序) + for (int i = keys.Length - 1; i >= 0; i--) + { + byte vk = GetVirtualKeyCode(keys[i]); + if (vk != 0) + { + keybd_event(vk, 0, KEYEVENTF_KEYUP, 0); + } + } + + return $"Success: 执行快捷键 {hotkey}"; + } + catch (Exception ex) + { + return $"Error: 快捷键失败 - {ex.Message}"; + } + } + + /// + /// 10. 关闭窗口 - Alt+F4 + /// + public static string CloseWindow(float x, float y) + { + try + { + MouseClick(x, y); + Thread.Sleep(100); + + keybd_event(VK_ALT, 0, KEYEVENTF_KEYDOWN, 0); + keybd_event(VK_F4, 0, KEYEVENTF_KEYDOWN, 0); + keybd_event(VK_F4, 0, KEYEVENTF_KEYUP, 0); + keybd_event(VK_ALT, 0, KEYEVENTF_KEYUP, 0); + + return "Success: 关闭窗口"; + } + catch (Exception ex) + { + return $"Error: 关闭窗口失败 - {ex.Message}"; + } + } + + private static byte GetVirtualKeyCode(string keyName) + { + return keyName.ToLowerInvariant() switch + { + "ctrl" or "control" => VK_CONTROL, + "shift" => VK_SHIFT, + "alt" => VK_ALT, + "enter" or "return" => VK_RETURN, + "esc" or "escape" => VK_ESCAPE, + "tab" => VK_TAB, + "backspace" or "back" => VK_BACK, + "win" or "windows" => VK_LWIN, + "f4" => VK_F4, + // 字母键 + "a" => 0x41, "b" => 0x42, "c" => 0x43, "d" => 0x44, "e" => 0x45, + "f" => 0x46, "g" => 0x47, "h" => 0x48, "i" => 0x49, "j" => 0x4A, + "k" => 0x4B, "l" => 0x4C, "m" => 0x4D, "n" => 0x4E, "o" => 0x4F, + "p" => 0x50, "q" => 0x51, "r" => 0x52, "s" => 0x53, "t" => 0x54, + "u" => 0x55, "v" => 0x56, "w" => 0x57, "x" => 0x58, "y" => 0x59, + "z" => 0x5A, + _ => 0 + }; + } + } +} diff --git a/Source/WulaFallenEmpire/WulaFallenEmpireMod.cs b/Source/WulaFallenEmpire/WulaFallenEmpireMod.cs index 28320e74..c7afb4e3 100644 --- a/Source/WulaFallenEmpire/WulaFallenEmpireMod.cs +++ b/Source/WulaFallenEmpire/WulaFallenEmpireMod.cs @@ -12,6 +12,7 @@ namespace WulaFallenEmpire { public static WulaFallenEmpireSettings settings; public static bool _showApiKey = false; + public static bool _showVlmApiKey = false; private string _maxContextTokensBuffer; public WulaFallenEmpireMod(ModContentPack content) : base(content) @@ -67,6 +68,37 @@ namespace WulaFallenEmpire listingStandard.GapLine(); listingStandard.CheckboxLabeled("Wula_EnableDebugLogs".Translate(), ref settings.enableDebugLogs, "Wula_EnableDebugLogsDesc".Translate()); + // VLM 设置部分 + listingStandard.GapLine(); + listingStandard.Label("VLM (视觉模型) 设置"); + + listingStandard.CheckboxLabeled("启用 VLM 视觉功能", ref settings.enableVlmFeatures, "启用后 AI 可以「看到」游戏屏幕并分析"); + + if (settings.enableVlmFeatures) + { + listingStandard.Label("VLM API Key:"); + Rect vlmKeyRect = listingStandard.GetRect(30f); + Rect vlmPasswordRect = new Rect(vlmKeyRect.x, vlmKeyRect.y, vlmKeyRect.width - toggleWidth - 5f, vlmKeyRect.height); + Rect vlmToggleRect = new Rect(vlmKeyRect.xMax - toggleWidth, vlmKeyRect.y, toggleWidth, vlmKeyRect.height); + + if (_showVlmApiKey) + { + settings.vlmApiKey = Widgets.TextField(vlmPasswordRect, settings.vlmApiKey ?? ""); + } + else + { + settings.vlmApiKey = GUI.PasswordField(vlmPasswordRect, settings.vlmApiKey ?? "", '•'); + } + Widgets.CheckboxLabeled(vlmToggleRect, "Show", ref _showVlmApiKey); + listingStandard.Gap(listingStandard.verticalSpacing); + + listingStandard.Label("VLM Base URL:"); + settings.vlmBaseUrl = listingStandard.TextEntry(settings.vlmBaseUrl ?? "https://dashscope.aliyuncs.com/compatible-mode/v1"); + + listingStandard.Label("VLM Model:"); + settings.vlmModel = listingStandard.TextEntry(settings.vlmModel ?? "qwen-vl-max"); + } + listingStandard.GapLine(); listingStandard.Label("Translation tools"); Rect exportRect = listingStandard.GetRect(30f); diff --git a/Tools/codex_handoff.md.resolved b/Tools/codex_handoff.md.resolved new file mode 100644 index 00000000..198fe358 --- /dev/null +++ b/Tools/codex_handoff.md.resolved @@ -0,0 +1,429 @@ +# RimWorld AI Agent 开发文档 + +> 本文档用于移交给 Codex 继续开发 + +--- + +## 1. 项目概述 + +### 目标 +创建一个**完全自主的 AI Agent**,能够自动玩 RimWorld 游戏。用户只需给出开放式指令(如"帮我挖点铁"或"帮我玩10分钟"),AI 即可独立决策并操作游戏。 + +### 技术栈 +- **语言**: C# (.NET Framework 4.8) +- **框架**: RimWorld Mod (Verse/RimWorld API) +- **AI 后端**: 阿里云百炼 (DashScope) API +- **VLM 模型**: Qwen-VL / Qwen-Omni-Realtime + +### 核心设计 +``` +用户指令 → AIIntelligenceCore → [被动模式 | 主动模式] → 工具执行 → 游戏操作 +``` + +--- + +## 2. 架构 + +### 2.1 模式切换设计 + +``` +┌────────────────────────────────────────────┐ +│ AIIntelligenceCore │ +│ ┌─────────────┐ ┌─────────────────┐ │ +│ │ 被动模式 │◄──►│ 主动模式 │ │ +│ │ (聊天对话) │ │ (Agent循环) │ │ +│ └─────────────┘ └────────┬────────┘ │ +└──────────────────────────────┼────────────┘ + ▼ + ┌─────────────────────┐ + │ AutonomousAgentLoop │ + │ Observe → Think │ + │ → Act │ + └─────────────────────┘ +``` + +**模式切换触发条件(待实现)**: +- 用户说"帮我玩X分钟" → 切换到主动模式 +- 主动模式任务完成 → 自动切回被动模式 +- 用户说"停止" → 强制切回被动模式 + +### 2.2 文件结构 + +``` +Source/WulaFallenEmpire/EventSystem/AI/ +├── AIIntelligenceCore.cs # 核心AI控制器(已有) +├── SimpleAIClient.cs # HTTP API 客户端(已有) +├── ScreenCaptureUtility.cs # 截屏工具(已有) +│ +├── Agent/ # ★ 新增目录 +│ ├── AutonomousAgentLoop.cs # 主动模式循环 +│ ├── StateObserver.cs # 游戏状态收集器 +│ ├── GameStateSnapshot.cs # 状态数据结构 +│ ├── VisualInteractionTools.cs # 视觉交互工具(10个) +│ ├── MouseSimulator.cs # 鼠标模拟 +│ └── OmniRealtimeClient.cs # WebSocket流式连接 +│ +├── Tools/ # AI工具 +│ ├── AITool.cs # 工具基类(已有) +│ ├── Tool_GetGameState.cs # ★ 新增 +│ ├── Tool_DesignateMine.cs # ★ 新增 +│ ├── Tool_DraftPawn.cs # ★ 新增 +│ ├── Tool_VisualClick.cs # ★ 新增 +│ └── ... (其他原有工具) +│ +└── UI/ + ├── Dialog_AIConversation.cs # 对话UI(已有) + └── Overlay_WulaLink.cs # 悬浮UI(已有) +``` + +--- + +## 3. 已完成组件 + +### 3.1 StateObserver (状态观察器) + +**路径**: [Agent/StateObserver.cs](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/StateObserver.cs) + +**功能**: 收集当前游戏状态,生成给 VLM 的文本描述 + +**API**: +```csharp +public static class StateObserver +{ + // 捕获当前游戏状态快照 + public static GameStateSnapshot CaptureState(); +} +``` + +**收集内容**: +- 时间(小时、季节、年份) +- 环境(生物群系、温度、天气) +- 殖民者(名字、健康、心情、当前工作、位置) +- 资源(钢铁、银、食物、医药等) +- 建筑进度(蓝图、建造框架) +- 威胁(敌对派系、距离) +- 最近消息 + +--- + +### 3.2 GameStateSnapshot (状态数据结构) + +**路径**: [Agent/GameStateSnapshot.cs](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/GameStateSnapshot.cs) + +**功能**: 存储游戏状态数据 + +**关键方法**: +```csharp +public class GameStateSnapshot +{ + public List Colonists; + public Dictionary Resources; + public List Threats; + + // 生成给VLM的文本描述 + public string ToPromptText(); +} +``` + +--- + +### 3.3 VisualInteractionTools (视觉交互工具集) + +**路径**: [Agent/VisualInteractionTools.cs](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs) + +**功能**: 10个纯视觉交互工具,使用 Windows API 模拟输入 + +| 方法 | 功能 | 参数 | +|------|------|------| +| [MouseClick(x, y, button, clicks)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#49-89) | 鼠标点击 | 0-1比例坐标 | +| [TypeText(x, y, text)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#90-118) | 输入文本 | 通过剪贴板 | +| [ScrollWindow(x, y, direction, amount)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#119-144) | 滚动 | up/down | +| [MouseDrag(sx, sy, ex, ey, duration)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#145-187) | 拖拽 | 起止坐标 | +| [Wait(seconds)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#188-203) | 等待 | 秒数 | +| [PressEnter()](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#204-220) | 按回车 | 无 | +| [PressEscape()](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#221-237) | 按ESC | 无 | +| [DeleteText(x, y, count)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#238-262) | 删除 | 字符数 | +| [PressHotkey(x, y, hotkey)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#263-306) | 快捷键 | 如"ctrl+c" | +| [CloseWindow(x, y)](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/VisualInteractionTools.cs#307-329) | 关闭窗口 | Alt+F4 | + +--- + +### 3.4 AutonomousAgentLoop (自主Agent循环) + +**路径**: [Agent/AutonomousAgentLoop.cs](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/AutonomousAgentLoop.cs) + +**功能**: 主动模式的核心循环 + +**状态**: +- `IsRunning`: 是否运行中 +- `CurrentObjective`: 当前目标 +- `DecisionCount`: 已执行决策次数 + +**关键API**: +```csharp +public class AutonomousAgentLoop : GameComponent +{ + public static AutonomousAgentLoop Instance; + + // 开始执行目标 + public void StartObjective(string objective); + + // 停止Agent + public void Stop(); + + // 事件 + public event Action OnDecisionMade; + public event Action OnObjectiveComplete; +} +``` + +**待完成**: [ExecuteDecision()](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/AutonomousAgentLoop.cs#244-269) 方法需要整合工具执行逻辑 + +--- + +### 3.5 原生API工具 + +| 工具 | 路径 | 功能 | 参数格式 | +|------|------|------|----------| +| [Tool_GetGameState](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Tools/Tool_GetGameState.cs#8-45) | Tools/ | 获取游戏状态 | `` | +| [Tool_DesignateMine](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Tools/Tool_DesignateMine.cs#12-139) | Tools/ | 采矿指令 | `数字数字可选` | +| [Tool_DraftPawn](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Tools/Tool_DraftPawn.cs#10-105) | Tools/ | 征召殖民者 | `名字true/false` | + +--- + +## 4. 待完成任务 + +### 4.1 模式切换整合 (高优先级) + +**目标**: 在 [AIIntelligenceCore](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/AIIntelligenceCore.cs#16-1364) 中实现被动/主动模式切换 + +**实现思路**: +```csharp +// AIIntelligenceCore 中添加 +private bool _isAgentMode = false; + +public void ProcessUserMessage(string message) +{ + // 检测是否触发主动模式 + if (IsAgentTrigger(message, out string objective, out float duration)) + { + _isAgentMode = true; + AutonomousAgentLoop.Instance.StartObjective(objective); + // 设置定时器,duration后自动停止 + } + else + { + // 正常对话处理 + RunConversation(message); + } +} + +private bool IsAgentTrigger(string msg, out string obj, out float dur) +{ + // 匹配模式: + // "帮我玩10分钟" → obj="管理殖民地", dur=600 + // "帮我挖点铁" → obj="采集铁矿", dur=0(无限) + // ... +} +``` + +--- + +### 4.2 工具执行整合 (高优先级) + +**目标**: 让 `AutonomousAgentLoop.ExecuteDecision()` 能够执行工具 + +**当前状态**: 方法体是空的 TODO + +**实现思路**: +```csharp +private void ExecuteDecision(string decision) +{ + // 1. 检查特殊标记 + if (decision.Contains(" Items = new List(); + + public void ExposeData() + { + Scribe_Collections.Look(ref Items, "todoItems", LookMode.Deep); + } +} + +public class TodoItem : IExposable +{ + public string Description; + public bool IsComplete; + public int Priority; + public int CreatedTick; +} +``` + +--- + +### 4.4 Qwen-Omni-Realtime 测试 (低优先级) + +**目标**: 测试 WebSocket 流式连接 + +**已完成**: [OmniRealtimeClient.cs](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/OmniRealtimeClient.cs) 基础实现 + +**待测试**: +- WebSocket 连接建立 +- 图片发送 (`input_image_buffer.append`) +- 文本接收 (`response.text.delta`) + +--- + +## 5. 关键接口参考 + +### 5.1 AITool 基类 + +```csharp +public abstract class AITool +{ + public abstract string Name { get; } + public abstract string Description { get; } + public abstract string UsageSchema { get; } + public abstract string Execute(string args); + + // 解析XML参数 + protected Dictionary ParseXmlArgs(string xmlContent); +} +``` + +### 5.2 SimpleAIClient API + +```csharp +public class SimpleAIClient +{ + public SimpleAIClient(string apiKey, string baseUrl, string model); + + // 文本对话 + public Task GetChatCompletionAsync( + string systemPrompt, + List<(string role, string message)> messages, + int maxTokens = 1024, + float temperature = 0.7f + ); + + // VLM 视觉分析 + public Task GetVisionCompletionAsync( + string systemPrompt, + string userPrompt, + string base64Image, + int maxTokens = 1024, + float temperature = 0.7f + ); +} +``` + +### 5.3 设置 (WulaFallenEmpireSettings) + +```csharp +public class WulaFallenEmpireSettings : ModSettings +{ + // 主模型 + public string apiKey; + public string baseUrl = "https://dashscope.aliyuncs.com/compatible-mode/v1"; + public string model = "qwen-turbo"; + + // VLM 模型 + public string vlmApiKey; + public string vlmBaseUrl; + public string vlmModel = "qwen-vl-max"; + public bool enableVlmFeatures = false; +} +``` + +--- + +## 6. 开发指南 + +### 6.1 添加新工具 + +1. 在 [Tools/](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/AIIntelligenceCore.cs#310-337) 创建新类继承 [AITool](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Tools/AITool.cs#8-41) +2. 实现 [Name](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/AIIntelligenceCore.cs#636-642), [Description](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Agent/StateObserver.cs#156-187), `UsageSchema`, [Execute](file:///C:/Steam/steamapps/common/RimWorld/Mods/3516260226/Source/WulaFallenEmpire/EventSystem/AI/Tools/Tool_VisualClick.cs#133-165) +3. 在 `AIIntelligenceCore.InitializeTools()` 中注册 + +```csharp +// 示例:Tool_BuildWall.cs +public class Tool_BuildWall : AITool +{ + public override string Name => "build_wall"; + public override string Description => "在指定位置放置墙壁蓝图"; + public override string UsageSchema => "XZ材料"; + + public override string Execute(string args) + { + var dict = ParseXmlArgs(args); + // 实现建造逻辑 + // GenConstruct.PlaceBlueprintForBuild(...) + return "Success: 墙壁蓝图已放置"; + } +} +``` + +### 6.2 调试技巧 + +- 使用 `WulaLog.Debug()` 输出日志 +- 检查 RimWorld 的 `Player.log` 文件 +- 在开发者模式下 `Prefs.DevMode = true` 显示更多信息 + +### 6.3 常见问题 + +**Q: .NET Framework 4.8 兼容性问题** +- 不支持 `TakeLast()` → 使用 `Skip(list.Count - n)` +- 不支持 `string.Contains(x, StringComparison)` → 使用 `IndexOf` + +**Q: Unity 主线程限制** +- 异步操作结果需要回到主线程执行 +- 使用 `LongEventHandler.ExecuteWhenFinished(() => { ... })` + +--- + +## 7. 参考资源 + +- [RimWorld Modding Wiki](https://rimworldwiki.com/wiki/Modding) +- [Harmony Patching](https://harmony.pardeike.net/) +- [阿里云百炼 API](https://help.aliyun.com/zh/model-studio/) +- [Qwen-Omni 文档](https://help.aliyun.com/zh/model-studio/user-guide/qwen-omni) + +--- + +**文档版本**: v1.0 +**更新时间**: 2025-12-27 +**作者**: Gemini AI Agent