RA3MysteriousResearchGroup/AnotherReplayReader
28
0
Fork 0
Code Issues Pull Requests Projects Releases 1 Wiki Activity

research

Browse Source
This commit is contained in:
lanyi
2026-08-23 03:31:42 +02:00
parent a2f0bcb371
commit 8e6d245c1c
3 changed files with 922 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AiV2.Tests/Program.cs
+664
View File
@@ -1,13 +1,20 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
using System.Text.Json;
using System.Text.Json.Serialization;
using System.Threading;
using System.Threading.Tasks;
using AnotherReplayReader;
using AnotherReplayReader.ReplayFile;
using AnotherReplayReader.Utils;
using AIAnalyze = AiV2.Tests.TestAIAnalyze;
namespace AiV2.Tests
{
@@ -36,6 +43,7 @@ namespace AiV2.Tests
Run("UserKnowledgeOverlay", UserKnowledgeOverlayTests.Run);
Run("PromptBuilders", PromptBuildersTests.Run);
Run("RevisionFactsAndSerialization", RevisionFactsAndSerializationTests.Run);
Run("OpenCodeGoFakeToolCallE2e", OpenCodeGoFakeToolCallTests.Run);
Run("UserReplayFactIndexRepro", UserReplayFactIndexReproTests.Run);
Console.WriteLine();
@@ -748,6 +756,662 @@ namespace AiV2.Tests
}
}
/// <summary>
/// AI 分析主工程实验代码已清理；本类只存在于测试工程中。
/// 用于直接构造 OpenAI 兼容请求，避免把实验字段带回主项目。
/// </summary>
internal class TestAIAnalyze
{
private readonly TestAiClient _client = new();
internal sealed class Result
{
public Result(string response, string? reasoning)
{
Response = response;
Reasoning = reasoning;
}
public string Response { get; }
public string? Reasoning { get; }
public int? PromptTokens { get; }
public int? CompletionTokens { get; }
public int? TotalTokens { get; }
public int? ReasoningTokens { get; }
}
internal sealed record ChatMessage(
[property: JsonPropertyName("role")] string Role,
[property: JsonPropertyName("content"),
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Content = null,
[property: JsonPropertyName("reasoning_content"),
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ReasoningContent = null,
[property: JsonPropertyName("tool_calls"),
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] IReadOnlyList<ChatMessage.ToolCall>? ToolCalls = null,
[property: JsonPropertyName("tool_call_id"),
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ToolCallId = null)
{
internal sealed record ToolCall(
[property: JsonPropertyName("id")] string Id,
[property: JsonPropertyName("type")] string Type,
[property: JsonPropertyName("function")] ToolCallFunction Function);
internal sealed record ToolCallFunction(
[property: JsonPropertyName("name")] string Name,
[property: JsonPropertyName("arguments")] string Arguments);
internal static ChatMessage Assistant(string? content, string? reasoningContent = null) =>
new("assistant",
string.IsNullOrEmpty(content) ? null : content,
string.IsNullOrEmpty(reasoningContent) ? null : reasoningContent);
internal static ChatMessage AssistantToolCall(
string? reasoningContent,
string toolCallId,
string toolName,
string arguments) =>
new(
"assistant",
null,
reasoningContent,
new[]
{
new ChatMessage.ToolCall(
toolCallId,
"function",
new ChatMessage.ToolCallFunction(toolName, arguments))
},
null);
internal static ChatMessage ToolResult(string toolCallId, string content) =>
new("tool", content, null, null, toolCallId);
}
public async Task<Result> CompleteAsync(
ImmutableList<ChatMessage> messages,
AiRequestContext context,
object? unused,
CancellationToken cancellationToken)
{
var response = await _client.CompleteAsync(
context.Provider, context.Model, messages, cancellationToken);
if (response.Error is not null)
{
throw new InvalidOperationException(response.Error);
}
return new Result(response.Content ?? string.Empty, response.Reasoning);
}
public static string BuildOverviewUserPrompt(IReadOnlyList<ReplaySlice> slices) =>
AnotherReplayReader.Utils.AIAnalyze.BuildOverviewUserPrompt(slices);
public static string BuildSegmentUserPromptV2(
int segmentIndex,
int totalSegments,
ReplaySlice slice,
int eventCount,
string? title,
string? description = null,
IEnumerable<string>? backqueryHints = null) =>
AnotherReplayReader.Utils.AIAnalyze.BuildSegmentUserPromptV2(
segmentIndex, totalSegments, slice, eventCount,
title, description, backqueryHints);
public static string BuildSummaryUserPromptV2(int totalEventCount) =>
AnotherReplayReader.Utils.AIAnalyze.BuildSummaryUserPromptV2(totalEventCount);
public static string BuildBackqueryUserPrompt(string sliceText) =>
AnotherReplayReader.Utils.AIAnalyze.BuildBackqueryUserPrompt(sliceText);
public static string BuildRevisionUserPrompt(
string draft,
string validationIssues,
string relevantFacts) =>
AnotherReplayReader.Utils.AIAnalyze.BuildRevisionUserPrompt(
draft, validationIssues, relevantFacts);
}
internal sealed record TestAiResponse(
string? Content,
string? Reasoning,
string? Error = null,
int? StatusCode = null);
internal sealed class TestAiClient
{
private readonly HttpClient _http = new()
{
Timeout = TimeSpan.FromMinutes(5)
};
public async Task<TestAiResponse> CompleteAsync(
AiProvider provider,
AiModel model,
IReadOnlyList<TestAIAnalyze.ChatMessage> messages,
CancellationToken cancellationToken)
{
var request = new Dictionary<string, object>
{
["model"] = model.ModelId,
["temperature"] = provider.DefaultTemperature,
["top_p"] = provider.DefaultTopP,
["max_tokens"] = provider.DefaultMaxTokens,
["stream"] = false
};
foreach (var kv in model.ExtraParameters)
{
request[kv.Key] = kv.Value;
}
request["messages"] = messages.ToArray();
var uri = new Uri(new(provider.BaseUrl.TrimEnd('/') + "/"), "chat/completions");
using var httpRequest = new HttpRequestMessage(HttpMethod.Post, uri);
httpRequest.Headers.Authorization = new AuthenticationHeaderValue(
"Bearer", provider.ApiKey);
httpRequest.Content = new StringContent(
JsonSerializer.Serialize(request), Encoding.UTF8, "application/json");
using var response = await _http.SendAsync(
httpRequest, cancellationToken);
var body = await response.Content.ReadAsStringAsync();
if (!response.IsSuccessStatusCode)
{
var errorMessage = body;
try
{
using var errorDocument = JsonDocument.Parse(body);
if (errorDocument.RootElement.TryGetProperty("error", out var error)
&& error.TryGetProperty("message", out var message))
{
errorMessage = message.GetString() ?? body;
}
}
catch
{
// 保留原始响应体。
}
return new TestAiResponse(
null, null, errorMessage, (int)response.StatusCode);
}
string? content = null;
string? reasoning = null;
using var document = JsonDocument.Parse(body);
AppendFromResponse(document.RootElement, ref content, ref reasoning);
return new TestAiResponse(
content, reasoning, null, (int)response.StatusCode);
}
private static void AppendFromResponse(
JsonElement root,
ref string? content,
ref string? reasoning)
{
if (!root.TryGetProperty("choices", out var choices)
|| choices.ValueKind != JsonValueKind.Array
|| choices.GetArrayLength() == 0)
{
return;
}
var message = choices[0].GetProperty("message");
if (message.TryGetProperty("content", out var contentProperty)
&& contentProperty.ValueKind == JsonValueKind.String)
{
content = contentProperty.GetString();
}
if (message.TryGetProperty("reasoning_content", out var reasoningProperty)
&& reasoningProperty.ValueKind == JsonValueKind.String)
{
reasoning = reasoningProperty.GetString();
}
}
}
internal static class OpenCodeGoFakeToolCallTests
{
private const string SystemPrompt =
"你是用于验证思维链回传的实验辅助。请先用 reasoning_content 进行内部推理，再给出简短、准确的回答。";
private const string FirstUserPrompt =
"请分析：A 比 B 高 20%，B 比 C 高 25%，那么 A 比 C 高多少？请先思考，再回答最终百分比。";
private const string ToolName = "analysis_hint";
private const string MemoryMarker = "TOKEN_MARK=731942";
public static void Run()
{
if (Environment.GetEnvironmentVariable("ARR_AI_E2E") != "1"
|| Environment.GetEnvironmentVariable("ARR_AI_E2E_TOOL") != "1")
{
Console.WriteLine(
" [跳过] 未同时设置 ARR_AI_E2E=1 与 ARR_AI_E2E_TOOL=1，"
+ "跳过伪造 tool call 历史实验");
return;
}
RunAsync().GetAwaiter().GetResult();
}
private static async Task RunAsync()
{
var settingsPath = FindSettingsPath();
Console.WriteLine($" [E2E-TOOL] 配置：{settingsPath}");
var settings = JsonSerializer.Deserialize<AiSettings>(
File.ReadAllText(settingsPath, Encoding.UTF8))
?? throw new InvalidOperationException("AI 设置文件无法解析");
var provider = settings.Providers.FirstOrDefault(p =>
p.Name.Equals("OpenCodeGo", StringComparison.OrdinalIgnoreCase)
&& p.Models.Any(m =>
m.ModelId.Equals("deepseek-v4-flash", StringComparison.OrdinalIgnoreCase)))
?? throw new InvalidOperationException(
"当前配置中未找到 OpenCodeGo / deepseek-v4-flash 服务与模型");
var model = provider.Models.First(m =>
m.ModelId.Equals("deepseek-v4-flash", StringComparison.OrdinalIgnoreCase));
if (string.IsNullOrWhiteSpace(provider.ApiKey))
{
throw new InvalidOperationException("OpenCodeGo 的 API Key 为空");
}
Console.WriteLine(
$" [E2E-TOOL] Provider={provider.Name}; Model={model.ModelId}; "
+ $"BaseUrl={provider.BaseUrl}; Stream={model.IsStream}; ApiKey=***");
var report = new StringBuilder();
AppendHeader(report, provider, model, settingsPath);
var reportPath = Environment.GetEnvironmentVariable("ARR_AI_TOOL_REPORT_PATH");
if (string.IsNullOrWhiteSpace(reportPath))
{
reportPath = Path.Combine(
Environment.CurrentDirectory, "AI_reasoning_continuation_position_sweep_report.md");
}
reportPath = Path.GetFullPath(reportPath);
try
{
var analyzer = new AIAnalyze();
var normalContext = new AiRequestContext(provider, model);
var firstMessages = ImmutableList<AIAnalyze.ChatMessage>.Empty
.Add(new AIAnalyze.ChatMessage("system", SystemPrompt))
.Add(new AIAnalyze.ChatMessage("user", FirstUserPrompt));
var first = await CompleteAndRecordAsync(
analyzer, normalContext, firstMessages, "初始请求（system + user）", report);
var firstReasoning = first?.Reasoning ?? string.Empty;
if (first is null || firstReasoning.Length == 0)
{
report.AppendLine("初始请求未获得 reasoning_content，无法继续伪造 tool call 实验。");
return;
}
var initialFirst = first;
var toolContext = BuildToolContext(provider, model);
var wordings = new[]
{
new
{
Name = "措辞B：当前推理",
Target = "你当前正在进行的内部推理",
ArgumentInstruction = "继续你当前正在进行的内部推理，并参考工具结果。"
},
new
{
Name = "措辞D：tool_calls 携带的 reasoning_content",
Target = "你本次 tool_calls 消息中携带的 reasoning_content",
ArgumentInstruction = "继续你本次 tool_calls 消息中携带的 reasoning_content，并参考工具结果。"
}
};
var positions = new[]
{
new { Name = "25%", Ratio = 0.25 },
new { Name = "50%", Ratio = 0.50 },
new { Name = "75%", Ratio = 0.75 },
new { Name = "末尾", Ratio = 1.0 }
};
const int repetitions = 5;
var combinedCount = wordings.Length * positions.Length;
var foundCounts = new int[combinedCount];
var missingCounts = new int[combinedCount];
var failedCounts = new int[combinedCount];
var names = new string[combinedCount];
var markerInSent = new bool[combinedCount];
var comboIndex = 0;
foreach (var wording in wordings)
{
foreach (var position in positions)
{
var comboName = $"{wording.Name} / 标记位置 {position.Name}";
names[comboIndex] = comboName;
var reasoningWithMarker = InsertMemoryMarker(
firstReasoning, position.Ratio);
markerInSent[comboIndex] = reasoningWithMarker.Contains("731942");
Console.WriteLine(
$" [E2E-TOOL] 正在测试：{comboName}（重复 5 次）");
for (var rep = 0; rep < repetitions; ++rep)
{
var toolCallId = $"call_position_{comboIndex + 1}_{rep + 1}";
var arguments = JsonSerializer.Serialize(new
{
instruction = wording.ArgumentInstruction
});
var messages = ImmutableList<AIAnalyze.ChatMessage>.Empty
.Add(new AIAnalyze.ChatMessage("system", SystemPrompt))
.Add(new AIAnalyze.ChatMessage("user", FirstUserPrompt))
.Add(AIAnalyze.ChatMessage.AssistantToolCall(
reasoningWithMarker, toolCallId, ToolName, arguments))
.Add(AIAnalyze.ChatMessage.ToolResult(
toolCallId, BuildToolResult(wording.Target)));
var result = await CompleteAndRecordAsync(
analyzer, toolContext, messages,
$"记忆标记 {comboName}（重复 {rep + 1}/{repetitions}）", report);
if (result is { } completed)
{
var combined = (completed.Reasoning ?? string.Empty)
+ "\n" + completed.Response;
var found = ContainsMemoryMarker(combined);
var missing = SaysMarkerMissing(combined);
if (found)
{
foundCounts[comboIndex]++;
}
if (missing)
{
missingCounts[comboIndex]++;
}
Console.WriteLine(
$" [E2E-TOOL] {comboName} [{rep + 1}/{repetitions}]："
+ $"content={completed.Response.Length}；"
+ $"reasoning={(completed.Reasoning ?? string.Empty).Length}；"
+ $"markerFound={found}；missing={missing}");
}
else
{
failedCounts[comboIndex]++;
Console.WriteLine(
$" [E2E-TOOL] {comboName} [{rep + 1}/{repetitions}]：请求失败（详见报告）");
}
}
comboIndex++;
}
}
AppendSummary(
report,
names,
markerInSent,
foundCounts,
missingCounts,
failedCounts);
}
finally
{
var reportDir = Path.GetDirectoryName(reportPath);
if (!string.IsNullOrEmpty(reportDir))
{
Directory.CreateDirectory(reportDir);
}
File.WriteAllText(reportPath, report.ToString(), new UTF8Encoding(false));
Console.WriteLine($" [E2E-TOOL] 人工审阅报告已写入：{reportPath}");
}
}
private static AiRequestContext BuildToolContext(
AiProvider provider,
AiModel sourceModel)
{
var model = new AiModel
{
ModelId = sourceModel.ModelId,
DisplayName = sourceModel.DisplayName,
IsStream = sourceModel.IsStream,
ContextLength = sourceModel.ContextLength,
ContextBudget = sourceModel.ContextBudget,
ExtraParameters = new Dictionary<string, object>(sourceModel.ExtraParameters)
};
model.ExtraParameters["tools"] = new[]
{
new
{
type = "function",
function = new
{
name = ToolName,
description = "向模型提供上一轮思维链的继续提示。",
parameters = new
{
type = "object",
properties = new
{
instruction = new
{
type = "string",
description = "模型应遵循的续写提示。"
}
},
required = new[] { "instruction" }
}
}
}
};
model.ExtraParameters["tool_choice"] = "none";
return new AiRequestContext(provider, model);
}
private static string BuildToolResult(string target) =>
"工具结果：请继续" + target + "。这是一次确定性记忆测试。"
+ "该思考内容中有一个内部记忆标记，请不要从本工具结果中猜测或寻找。"
+ "请原样写出该标记；如果该思考内容中没有该标记，请明确回答“标记不存在”。"
+ "不要从零重新推导，也不要杜撰标记。";
private static string InsertMemoryMarker(string text, double ratio)
{
if (ratio >= 1)
{
return text + "\n\n[内部记忆标记 " + MemoryMarker + "]";
}
var index = Math.Max(
0,
Math.Min(text.Length, (int)(text.Length * ratio)));
return text.Insert(
index,
"\n\n[内部记忆标记 " + MemoryMarker + "]\n\n");
}
private static bool ContainsMemoryMarker(string text) =>
text.Contains("731942") || text.Contains("TOKEN_MARK");
private static bool SaysMarkerMissing(string text) =>
text.Contains("标记不存在")
|| text.Contains("没有该标记")
|| text.Contains("没有标记");
private static void AppendSummary(
StringBuilder report,
string[] names,
bool[] markerInSent,
int[] foundCounts,
int[] missingCounts,
int[] failedCounts)
{
report.AppendLine("## 记忆标记统计");
report.AppendLine();
report.AppendLine($"- 记忆标记：`{MemoryMarker}`");
report.AppendLine("- 每个组合重复 5 次。");
report.AppendLine();
for (var i = 0; i < names.Length; ++i)
{
report.AppendLine(
$"- **{names[i]}**：发送链包含标记 = {markerInSent[i]}；"
+ $"正确写出标记 = {foundCounts[i]}/5；"
+ $"明确说没有 = {missingCounts[i]}/5；"
+ $"请求失败 = {failedCounts[i]}/5");
}
report.AppendLine();
}
private static async Task<AIAnalyze.Result?> CompleteAndRecordAsync(
AIAnalyze analyzer,
AiRequestContext context,
ImmutableList<AIAnalyze.ChatMessage> messages,
string label,
StringBuilder report)
{
var sw = Stopwatch.StartNew();
try
{
var result = await analyzer.CompleteAsync(
messages, context, null, CancellationToken.None);
sw.Stop();
AppendRecord(report, label, messages, result, sw.Elapsed);
return result;
}
catch (Exception ex)
{
sw.Stop();
report.AppendLine($"## {label}（请求失败）");
report.AppendLine();
report.AppendLine($"- 耗时：{sw.Elapsed.TotalSeconds:0.00} 秒");
report.AppendLine($"- 错误：{ex.Message}");
report.AppendLine("- 关键点：assistant 消息包含 tool_calls，随后有 tool 回复；请求携带 tools 定义。");
report.AppendLine();
report.AppendLine("---");
report.AppendLine();
Console.WriteLine($" [E2E-TOOL] {label} 失败：{ex.Message}");
return null;
}
}
private static void AppendHeader(
StringBuilder report,
AiProvider provider,
AiModel model,
string settingsPath)
{
report.AppendLine("# 伪造 tool call 历史实验");
report.AppendLine();
report.AppendLine($"- 配置：{provider.Name} / {model.ModelId}");
report.AppendLine($"- BaseUrl：{provider.BaseUrl}");
report.AppendLine($"- 配置文件：{settingsPath}");
report.AppendLine("- API Key：***（不写入报告）");
report.AppendLine();
report.AppendLine("本实验在请求中声明了 `tools` 并将 `tool_choice` 设为 `none`；");
report.AppendLine("构造的历史为：system → 初始 user → assistant(reasoning_content + tool_calls) → tool(tool_call_id + 提示结果)。");
report.AppendLine(
$"完整思维链按 25%/50%/75%/末尾四种位置注入确定性记忆标记 `{MemoryMarker}`；"
+ "工具结果使用“当前推理”和“本次 tool_calls 消息携带的 reasoning_content”两种措辞，"
+ "且不提及“上一轮”“截断点”等概念，要求模型原样写出标记。");
report.AppendLine("没有第 5 条新的 user 消息。");
report.AppendLine();
}
private static void AppendRecord(
StringBuilder report,
string label,
ImmutableList<AIAnalyze.ChatMessage> messages,
AIAnalyze.Result result,
TimeSpan elapsed)
{
report.AppendLine($"## {label}");
report.AppendLine();
report.AppendLine($"- 耗时：{elapsed.TotalSeconds:0.00} 秒");
report.AppendLine($"- prompt_tokens：{result.PromptTokens?.ToString() ?? "?"}");
report.AppendLine($"- completion_tokens：{result.CompletionTokens?.ToString() ?? "?"}");
report.AppendLine($"- reasoning_tokens：{result.ReasoningTokens?.ToString() ?? "?"}");
report.AppendLine();
report.AppendLine("### 发送的 messages");
report.AppendLine();
for (var i = 0; i < messages.Count; ++i)
{
var message = messages[i];
report.AppendLine($"**{i + 1}. role = `{message.Role}`**");
report.AppendLine();
if (message.Content is null)
{
report.AppendLine("content：`未发送`");
}
else
{
report.AppendLine("content：");
AppendTextBlock(report, message.Content);
}
report.AppendLine();
if (message.ToolCallId is not null)
{
report.AppendLine($"tool_call_id：`{message.ToolCallId}`");
report.AppendLine();
}
if (message.ToolCalls is { } toolCalls)
{
report.AppendLine("tool_calls：");
foreach (var toolCall in toolCalls)
{
report.AppendLine(
$"- id=`{toolCall.Id}`; type=`{toolCall.Type}`; "
+ $"name=`{toolCall.Function.Name}`");
report.AppendLine();
report.AppendLine("arguments：");
AppendTextBlock(report, toolCall.Function.Arguments);
}
report.AppendLine();
}
if (message.Role == "assistant")
{
report.AppendLine(
message.ReasoningContent is null
? "reasoning_content：`未发送`"
: "reasoning_content：");
if (message.ReasoningContent is not null)
{
AppendTextBlock(report, message.ReasoningContent);
}
report.AppendLine();
}
report.AppendLine();
}
report.AppendLine("### 模型响应");
report.AppendLine();
report.AppendLine("新 reasoning_content：");
AppendTextBlock(report, result.Reasoning ?? string.Empty);
report.AppendLine();
report.AppendLine("回答正文（content）：");
AppendTextBlock(report, result.Response);
report.AppendLine();
report.AppendLine("---");
report.AppendLine();
}
private static void AppendTextBlock(StringBuilder report, string? text)
{
report.AppendLine("```text");
report.AppendLine(string.IsNullOrEmpty(text) ? "(空)" : text!);
report.AppendLine("```");
}
private static string FindSettingsPath()
{
var explicitPath = Environment.GetEnvironmentVariable("ARR_AI_SETTINGS_PATH");
if (!string.IsNullOrWhiteSpace(explicitPath) && File.Exists(explicitPath))
{
return explicitPath;
}
var dir = new DirectoryInfo(AppContext.BaseDirectory);
for (var i = 0; i < 6 && dir is not null; ++i)
{
var candidate = Path.Combine(
dir.FullName, "bin", "Debug", "net461", "AnotherReplayReader.ai_settings.json");
if (File.Exists(candidate))
{
return candidate;
}
var directCandidate = Path.Combine(
dir.FullName, "AnotherReplayReader.ai_settings.json");
if (File.Exists(directCandidate))
{
return directCandidate;
}
dir = dir.Parent;
}
throw new InvalidOperationException(
"未找到 AnotherReplayReader.ai_settings.json；可用 ARR_AI_SETTINGS_PATH 指定");
}
}
internal static class UserReplayFactIndexReproTests
{
public static void Run()