remove reasoning guard
This commit is contained in:
+1
-803
@@ -46,9 +46,6 @@ namespace AiV2.Tests
|
||||
Run("UserKnowledgeOverlay", UserKnowledgeOverlayTests.Run);
|
||||
Run("PromptBuilders", PromptBuildersTests.Run);
|
||||
Run("RevisionFactsAndSerialization", RevisionFactsAndSerializationTests.Run);
|
||||
Run("ReasoningGuard", ReasoningGuardTests.Run);
|
||||
Run("OpenCodeGoFakeToolCallE2e", OpenCodeGoFakeToolCallTests.Run);
|
||||
Run("OpenCodeGoGuardE2e", OpenCodeGoGuardE2e.Run);
|
||||
Run("UserReplayFactIndexRepro", UserReplayFactIndexReproTests.Run);
|
||||
|
||||
Console.WriteLine();
|
||||
@@ -992,46 +989,12 @@ namespace AiV2.Tests
|
||||
[property: JsonPropertyName("content"),
|
||||
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? Content = null,
|
||||
[property: JsonPropertyName("reasoning_content"),
|
||||
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ReasoningContent = null,
|
||||
[property: JsonPropertyName("tool_calls"),
|
||||
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] IReadOnlyList<ChatMessage.ToolCall>? ToolCalls = null,
|
||||
[property: JsonPropertyName("tool_call_id"),
|
||||
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ToolCallId = null)
|
||||
JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] string? ReasoningContent = null)
|
||||
{
|
||||
internal sealed record ToolCall(
|
||||
[property: JsonPropertyName("id")] string Id,
|
||||
[property: JsonPropertyName("type")] string Type,
|
||||
[property: JsonPropertyName("function")] ToolCallFunction Function);
|
||||
|
||||
internal sealed record ToolCallFunction(
|
||||
[property: JsonPropertyName("name")] string Name,
|
||||
[property: JsonPropertyName("arguments")] string Arguments);
|
||||
|
||||
internal static ChatMessage Assistant(string? content, string? reasoningContent = null) =>
|
||||
new("assistant",
|
||||
string.IsNullOrEmpty(content) ? null : content,
|
||||
string.IsNullOrEmpty(reasoningContent) ? null : reasoningContent);
|
||||
|
||||
internal static ChatMessage AssistantToolCall(
|
||||
string? reasoningContent,
|
||||
string toolCallId,
|
||||
string toolName,
|
||||
string arguments) =>
|
||||
new(
|
||||
"assistant",
|
||||
null,
|
||||
reasoningContent,
|
||||
new[]
|
||||
{
|
||||
new ChatMessage.ToolCall(
|
||||
toolCallId,
|
||||
"function",
|
||||
new ChatMessage.ToolCallFunction(toolName, arguments))
|
||||
},
|
||||
null);
|
||||
|
||||
internal static ChatMessage ToolResult(string toolCallId, string content) =>
|
||||
new("tool", content, null, null, toolCallId);
|
||||
}
|
||||
|
||||
public async Task<Result> CompleteAsync(
|
||||
@@ -1189,771 +1152,6 @@ namespace AiV2.Tests
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class GuardHttpHandler : HttpMessageHandler
|
||||
{
|
||||
public int RequestCount;
|
||||
public string? FirstRequestBody;
|
||||
public string? SecondRequestBody;
|
||||
public bool FirstResponseContentOnly;
|
||||
public bool SecondResponseInterrupted;
|
||||
|
||||
protected override async Task<HttpResponseMessage> SendAsync(
|
||||
HttpRequestMessage request,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
var body = await request.Content!.ReadAsStringAsync();
|
||||
RequestCount++;
|
||||
if (RequestCount == 1)
|
||||
{
|
||||
FirstRequestBody = body;
|
||||
if (FirstResponseContentOnly)
|
||||
{
|
||||
return CreateSse(
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"final\"}}]}\n",
|
||||
"data: [DONE]\n");
|
||||
}
|
||||
return CreateSse(
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"partial first\"}}]}\n",
|
||||
"data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"reasoning\"}}]}\n");
|
||||
}
|
||||
|
||||
SecondRequestBody = body;
|
||||
if (SecondResponseInterrupted)
|
||||
{
|
||||
return CreateSse(
|
||||
"data: {\"choices\":[{\"delta\":{\"reasoning_content\":\"second reasoning\"}}]}\n");
|
||||
}
|
||||
return CreateSse(
|
||||
"data: {\"choices\":[{\"delta\":{\"content\":\"final\"}}]}\n",
|
||||
"data: [DONE]\n");
|
||||
}
|
||||
|
||||
private static HttpResponseMessage CreateSse(params string[] lines)
|
||||
{
|
||||
return new HttpResponseMessage(System.Net.HttpStatusCode.OK)
|
||||
{
|
||||
Content = new StringContent(
|
||||
string.Concat(lines),
|
||||
Encoding.UTF8,
|
||||
"text/event-stream")
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
internal static class ReasoningGuardTests
|
||||
{
|
||||
public static void Run()
|
||||
{
|
||||
TestPrepareReasoning();
|
||||
TestToolPrompt();
|
||||
TestMessageSerialization();
|
||||
TestDisabledAsync().GetAwaiter().GetResult();
|
||||
TestGuardedContinuationAsync().GetAwaiter().GetResult();
|
||||
TestFallbackAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
private static AiProvider CreateProvider() =>
|
||||
new()
|
||||
{
|
||||
Name = "测试",
|
||||
BaseUrl = "http://localhost/v1",
|
||||
ApiKey = "test-key",
|
||||
DefaultMaxTokens = 16384
|
||||
};
|
||||
|
||||
private static AiModel CreateModel(bool enabled = true) =>
|
||||
new()
|
||||
{
|
||||
ModelId = "deepseek-v4-flash",
|
||||
IsStream = true,
|
||||
ContextLength = 1_000_000,
|
||||
ContextBudget = 160_000,
|
||||
ReasoningGuardEnabled = enabled,
|
||||
ReasoningGuardTokenLimit = enabled ? 1 : null,
|
||||
ExtraParameters = new Dictionary<string, object>()
|
||||
};
|
||||
|
||||
private static ImmutableList<MainAIAnalyze.ChatMessage> CreateMessages() =>
|
||||
ImmutableList<MainAIAnalyze.ChatMessage>.Empty
|
||||
.Add(new MainAIAnalyze.ChatMessage("system", "测试系统"))
|
||||
.Add(new MainAIAnalyze.ChatMessage("user", "请分析。"));
|
||||
|
||||
private static async Task<MainAIAnalyze.Result> CompleteAsync(
|
||||
GuardHttpHandler handler,
|
||||
AiModel model)
|
||||
{
|
||||
using var http = new HttpClient(handler);
|
||||
var analyzer = new MainAIAnalyze(http);
|
||||
return await analyzer.CompleteAsync(
|
||||
CreateMessages(),
|
||||
new AiRequestContext(CreateProvider(), model),
|
||||
_ => { },
|
||||
CancellationToken.None);
|
||||
}
|
||||
|
||||
private static void TestPrepareReasoning()
|
||||
{
|
||||
var shortReasoning = "第一行。\n第二行。\n第三行。";
|
||||
var prepared = AiReasoningGuard.PrepareReasoning(shortReasoning, 1000);
|
||||
Program.Assert(
|
||||
prepared.Text.TrimEnd().EndsWith(AiReasoningGuard.WrapUpStatement),
|
||||
"末尾收尾语句");
|
||||
Program.Assert(
|
||||
prepared.Text.IndexOf(AiReasoningGuard.TruncationMarker) >= 0
|
||||
&& prepared.Text.IndexOf(AiReasoningGuard.TruncationMarker)
|
||||
== prepared.Text.LastIndexOf(AiReasoningGuard.TruncationMarker),
|
||||
"只追加一次截断标记");
|
||||
Program.Assert(
|
||||
prepared.Text.IndexOf(AiReasoningGuard.TruncationMarker)
|
||||
< prepared.Text.IndexOf(AiReasoningGuard.WrapUpStatement),
|
||||
"截断标记在收尾语句之前");
|
||||
|
||||
var longReasoning = "前文。\n" + new string('x', 500) + "后文。";
|
||||
var truncated = AiReasoningGuard.PrepareReasoning(longReasoning, 10);
|
||||
Program.Assert(!truncated.Text.Contains("后文"), "超限部分被截掉");
|
||||
|
||||
var formatState = AiReasoningGuard.DescribeFormatState(
|
||||
"```json\n{\"unitClaims\":[");
|
||||
Program.Assert(
|
||||
formatState.Contains("代码围栏未闭合")
|
||||
&& formatState.Contains("JSON"),
|
||||
"格式状态检测");
|
||||
}
|
||||
|
||||
private static void TestToolPrompt()
|
||||
{
|
||||
var result = AiReasoningGuard.BuildToolResult();
|
||||
Program.Assert(
|
||||
result.Contains("立即停止继续展开推理")
|
||||
&& result.Contains("直接输出最终结果")
|
||||
&& result.Contains("最后一句已经宣告收尾"),
|
||||
"强收尾措辞");
|
||||
Program.Assert(
|
||||
!result.Contains("上一轮思维链")
|
||||
&& !result.Contains("截断点")
|
||||
&& !result.Contains("My reasoning has been truncated")
|
||||
&& !result.Contains("token limit"),
|
||||
"避免歧义措辞");
|
||||
}
|
||||
|
||||
private static void TestMessageSerialization()
|
||||
{
|
||||
var assistant = MainAIAnalyze.ChatMessage.AssistantToolCall(
|
||||
"thinking",
|
||||
"call_1",
|
||||
"analysis_hint",
|
||||
"{\"instruction\":\"wrap up\"}");
|
||||
var assistantJson = JsonSerializer.Serialize(assistant);
|
||||
Program.Assert(
|
||||
assistantJson.Contains("\"reasoning_content\":\"thinking\"")
|
||||
&& assistantJson.Contains("\"tool_calls\"")
|
||||
&& assistantJson.Contains("\"analysis_hint\""),
|
||||
"assistant 工具历史序列化");
|
||||
|
||||
var tool = MainAIAnalyze.ChatMessage.ToolResult(
|
||||
"call_1",
|
||||
AiReasoningGuard.BuildToolResult());
|
||||
var toolJson = JsonSerializer.Serialize(tool);
|
||||
Program.Assert(
|
||||
toolJson.Contains("\"tool_call_id\":\"call_1\""),
|
||||
"tool 结果序列化");
|
||||
}
|
||||
|
||||
private static async Task TestDisabledAsync()
|
||||
{
|
||||
var handler = new GuardHttpHandler { FirstResponseContentOnly = true };
|
||||
var result = await CompleteAsync(handler, CreateModel(enabled: false));
|
||||
Program.AssertEqual(1, handler.RequestCount, "未启用时只发一次请求");
|
||||
Program.Assert(!result.ContinuationApplied, "未启用不续写");
|
||||
Program.AssertEqual("final", result.Response, "未启用保持普通结果");
|
||||
}
|
||||
|
||||
private static async Task TestGuardedContinuationAsync()
|
||||
{
|
||||
var handler = new GuardHttpHandler();
|
||||
var result = await CompleteAsync(handler, CreateModel());
|
||||
Program.AssertEqual(2, handler.RequestCount, "触发后发起一次续写");
|
||||
Program.Assert(result.ContinuationApplied, "续写成功");
|
||||
Program.AssertEqual("final", result.Response, "续写返回最终正文");
|
||||
Program.Assert(
|
||||
handler.FirstRequestBody is { } firstBody
|
||||
&& !firstBody.Contains("\"tools\""),
|
||||
"首次请求不携带 tool 定义");
|
||||
Program.Assert(
|
||||
handler.SecondRequestBody is { } secondBody
|
||||
&& secondBody.Contains("\"tools\"")
|
||||
&& secondBody.Contains("\"tool_choice\":\"none\"")
|
||||
&& secondBody.Contains("\"reasoning_content\"")
|
||||
&& secondBody.Contains("\"tool_calls\"")
|
||||
&& secondBody.Contains("\"tool_call_id\""),
|
||||
"续写请求携带工具历史");
|
||||
Program.Assert(
|
||||
result.ReasoningOriginalRequestJson is { } originalJson
|
||||
&& originalJson.Contains("\"messages\""),
|
||||
"首次请求 JSON 回传 UI");
|
||||
Program.Assert(
|
||||
result.ReasoningContinuationRequestJson is { } continuationJson
|
||||
&& continuationJson.Contains("\"tools\"")
|
||||
&& continuationJson.Contains("\"reasoning_content\"")
|
||||
&& continuationJson.Contains("我已经整理出足够的信息"),
|
||||
"修改后请求 JSON 回传 UI");
|
||||
}
|
||||
|
||||
private static async Task TestFallbackAsync()
|
||||
{
|
||||
var handler = new GuardHttpHandler { SecondResponseInterrupted = true };
|
||||
var result = await CompleteAsync(handler, CreateModel());
|
||||
Program.Assert(!result.ContinuationApplied, "二次超限不再续写");
|
||||
Program.Assert(result.ReasoningInterrupted, "保留中断状态");
|
||||
Program.Assert(
|
||||
result.ReasoningContinuationError is { } error
|
||||
&& error.Contains("回退"),
|
||||
"回退提示");
|
||||
Program.Assert(
|
||||
result.ReasoningContinuationRequestJson is { } continuationJson
|
||||
&& continuationJson.Contains("\"tools\""),
|
||||
"回退时仍提供修改后请求 JSON");
|
||||
Program.AssertEqual("partial first", result.Response, "回退首次部分正文");
|
||||
}
|
||||
}
|
||||
|
||||
internal static class OpenCodeGoFakeToolCallTests
|
||||
{
|
||||
private const string SystemPrompt =
|
||||
"你是用于验证思维链回传的实验辅助。请先用 reasoning_content 进行内部推理，再给出简短、准确的回答。";
|
||||
private const string FirstUserPrompt =
|
||||
"请分析：A 比 B 高 20%，B 比 C 高 25%，那么 A 比 C 高多少？请先思考，再回答最终百分比。";
|
||||
private const string ToolName = "analysis_hint";
|
||||
private const string MemoryMarker = "TOKEN_MARK=731942";
|
||||
|
||||
public static void Run()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("ARR_AI_E2E") != "1"
|
||||
|| Environment.GetEnvironmentVariable("ARR_AI_E2E_TOOL") != "1")
|
||||
{
|
||||
Console.WriteLine(
|
||||
" [跳过] 未同时设置 ARR_AI_E2E=1 与 ARR_AI_E2E_TOOL=1，"
|
||||
+ "跳过伪造 tool call 历史实验");
|
||||
return;
|
||||
}
|
||||
RunAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
private static async Task RunAsync()
|
||||
{
|
||||
var settingsPath = FindSettingsPath();
|
||||
Console.WriteLine($" [E2E-TOOL] 配置：{settingsPath}");
|
||||
|
||||
var settings = JsonSerializer.Deserialize<AiSettings>(
|
||||
File.ReadAllText(settingsPath, Encoding.UTF8))
|
||||
?? throw new InvalidOperationException("AI 设置文件无法解析");
|
||||
var provider = settings.Providers.FirstOrDefault(p =>
|
||||
p.Name.Equals("OpenCodeGo", StringComparison.OrdinalIgnoreCase)
|
||||
&& p.Models.Any(m =>
|
||||
m.ModelId.Equals("deepseek-v4-flash", StringComparison.OrdinalIgnoreCase)))
|
||||
?? throw new InvalidOperationException(
|
||||
"当前配置中未找到 OpenCodeGo / deepseek-v4-flash 服务与模型");
|
||||
var model = provider.Models.First(m =>
|
||||
m.ModelId.Equals("deepseek-v4-flash", StringComparison.OrdinalIgnoreCase));
|
||||
if (string.IsNullOrWhiteSpace(provider.ApiKey))
|
||||
{
|
||||
throw new InvalidOperationException("OpenCodeGo 的 API Key 为空");
|
||||
}
|
||||
|
||||
Console.WriteLine(
|
||||
$" [E2E-TOOL] Provider={provider.Name}; Model={model.ModelId}; "
|
||||
+ $"BaseUrl={provider.BaseUrl}; Stream={model.IsStream}; ApiKey=***");
|
||||
|
||||
var report = new StringBuilder();
|
||||
AppendHeader(report, provider, model, settingsPath);
|
||||
var reportPath = Environment.GetEnvironmentVariable("ARR_AI_TOOL_REPORT_PATH");
|
||||
if (string.IsNullOrWhiteSpace(reportPath))
|
||||
{
|
||||
reportPath = Path.Combine(
|
||||
Environment.CurrentDirectory, "AI_reasoning_continuation_position_sweep_report.md");
|
||||
}
|
||||
reportPath = Path.GetFullPath(reportPath);
|
||||
|
||||
try
|
||||
{
|
||||
var analyzer = new AIAnalyze();
|
||||
var normalContext = new AiRequestContext(provider, model);
|
||||
var firstMessages = ImmutableList<AIAnalyze.ChatMessage>.Empty
|
||||
.Add(new AIAnalyze.ChatMessage("system", SystemPrompt))
|
||||
.Add(new AIAnalyze.ChatMessage("user", FirstUserPrompt));
|
||||
var first = await CompleteAndRecordAsync(
|
||||
analyzer, normalContext, firstMessages, "初始请求（system + user）", report);
|
||||
var firstReasoning = first?.Reasoning ?? string.Empty;
|
||||
if (first is null || firstReasoning.Length == 0)
|
||||
{
|
||||
report.AppendLine("初始请求未获得 reasoning_content，无法继续伪造 tool call 实验。");
|
||||
return;
|
||||
}
|
||||
var initialFirst = first;
|
||||
|
||||
var toolContext = BuildToolContext(provider, model);
|
||||
var wordings = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
Name = "措辞B：当前推理",
|
||||
Target = "你当前正在进行的内部推理",
|
||||
ArgumentInstruction = "继续你当前正在进行的内部推理，并参考工具结果。"
|
||||
},
|
||||
new
|
||||
{
|
||||
Name = "措辞D：tool_calls 携带的 reasoning_content",
|
||||
Target = "你本次 tool_calls 消息中携带的 reasoning_content",
|
||||
ArgumentInstruction = "继续你本次 tool_calls 消息中携带的 reasoning_content，并参考工具结果。"
|
||||
}
|
||||
};
|
||||
var positions = new[]
|
||||
{
|
||||
new { Name = "25%", Ratio = 0.25 },
|
||||
new { Name = "50%", Ratio = 0.50 },
|
||||
new { Name = "75%", Ratio = 0.75 },
|
||||
new { Name = "末尾", Ratio = 1.0 }
|
||||
};
|
||||
const int repetitions = 5;
|
||||
var combinedCount = wordings.Length * positions.Length;
|
||||
var foundCounts = new int[combinedCount];
|
||||
var missingCounts = new int[combinedCount];
|
||||
var failedCounts = new int[combinedCount];
|
||||
var names = new string[combinedCount];
|
||||
var markerInSent = new bool[combinedCount];
|
||||
var comboIndex = 0;
|
||||
|
||||
foreach (var wording in wordings)
|
||||
{
|
||||
foreach (var position in positions)
|
||||
{
|
||||
var comboName = $"{wording.Name} / 标记位置 {position.Name}";
|
||||
names[comboIndex] = comboName;
|
||||
var reasoningWithMarker = InsertMemoryMarker(
|
||||
firstReasoning, position.Ratio);
|
||||
markerInSent[comboIndex] = reasoningWithMarker.Contains("731942");
|
||||
Console.WriteLine(
|
||||
$" [E2E-TOOL] 正在测试：{comboName}（重复 5 次）");
|
||||
for (var rep = 0; rep < repetitions; ++rep)
|
||||
{
|
||||
var toolCallId = $"call_position_{comboIndex + 1}_{rep + 1}";
|
||||
var arguments = JsonSerializer.Serialize(new
|
||||
{
|
||||
instruction = wording.ArgumentInstruction
|
||||
});
|
||||
var messages = ImmutableList<AIAnalyze.ChatMessage>.Empty
|
||||
.Add(new AIAnalyze.ChatMessage("system", SystemPrompt))
|
||||
.Add(new AIAnalyze.ChatMessage("user", FirstUserPrompt))
|
||||
.Add(AIAnalyze.ChatMessage.AssistantToolCall(
|
||||
reasoningWithMarker, toolCallId, ToolName, arguments))
|
||||
.Add(AIAnalyze.ChatMessage.ToolResult(
|
||||
toolCallId, BuildToolResult(wording.Target)));
|
||||
var result = await CompleteAndRecordAsync(
|
||||
analyzer, toolContext, messages,
|
||||
$"记忆标记 {comboName}（重复 {rep + 1}/{repetitions}）", report);
|
||||
if (result is { } completed)
|
||||
{
|
||||
var combined = (completed.Reasoning ?? string.Empty)
|
||||
+ "\n" + completed.Response;
|
||||
var found = ContainsMemoryMarker(combined);
|
||||
var missing = SaysMarkerMissing(combined);
|
||||
if (found)
|
||||
{
|
||||
foundCounts[comboIndex]++;
|
||||
}
|
||||
if (missing)
|
||||
{
|
||||
missingCounts[comboIndex]++;
|
||||
}
|
||||
Console.WriteLine(
|
||||
$" [E2E-TOOL] {comboName} [{rep + 1}/{repetitions}]："
|
||||
+ $"content={completed.Response.Length}；"
|
||||
+ $"reasoning={(completed.Reasoning ?? string.Empty).Length}；"
|
||||
+ $"markerFound={found}；missing={missing}");
|
||||
}
|
||||
else
|
||||
{
|
||||
failedCounts[comboIndex]++;
|
||||
Console.WriteLine(
|
||||
$" [E2E-TOOL] {comboName} [{rep + 1}/{repetitions}]：请求失败（详见报告）");
|
||||
}
|
||||
}
|
||||
comboIndex++;
|
||||
}
|
||||
}
|
||||
AppendSummary(
|
||||
report,
|
||||
names,
|
||||
markerInSent,
|
||||
foundCounts,
|
||||
missingCounts,
|
||||
failedCounts);
|
||||
}
|
||||
finally
|
||||
{
|
||||
var reportDir = Path.GetDirectoryName(reportPath);
|
||||
if (!string.IsNullOrEmpty(reportDir))
|
||||
{
|
||||
Directory.CreateDirectory(reportDir);
|
||||
}
|
||||
File.WriteAllText(reportPath, report.ToString(), new UTF8Encoding(false));
|
||||
Console.WriteLine($" [E2E-TOOL] 人工审阅报告已写入：{reportPath}");
|
||||
}
|
||||
}
|
||||
|
||||
private static AiRequestContext BuildToolContext(
|
||||
AiProvider provider,
|
||||
AiModel sourceModel)
|
||||
{
|
||||
var model = new AiModel
|
||||
{
|
||||
ModelId = sourceModel.ModelId,
|
||||
DisplayName = sourceModel.DisplayName,
|
||||
IsStream = sourceModel.IsStream,
|
||||
ContextLength = sourceModel.ContextLength,
|
||||
ContextBudget = sourceModel.ContextBudget,
|
||||
ExtraParameters = new Dictionary<string, object>(sourceModel.ExtraParameters)
|
||||
};
|
||||
model.ExtraParameters["tools"] = new[]
|
||||
{
|
||||
new
|
||||
{
|
||||
type = "function",
|
||||
function = new
|
||||
{
|
||||
name = ToolName,
|
||||
description = "向模型提供上一轮思维链的继续提示。",
|
||||
parameters = new
|
||||
{
|
||||
type = "object",
|
||||
properties = new
|
||||
{
|
||||
instruction = new
|
||||
{
|
||||
type = "string",
|
||||
description = "模型应遵循的续写提示。"
|
||||
}
|
||||
},
|
||||
required = new[] { "instruction" }
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
model.ExtraParameters["tool_choice"] = "none";
|
||||
return new AiRequestContext(provider, model);
|
||||
}
|
||||
|
||||
private static string BuildToolResult(string target) =>
|
||||
"工具结果：请继续" + target + "。这是一次确定性记忆测试。"
|
||||
+ "该思考内容中有一个内部记忆标记，请不要从本工具结果中猜测或寻找。"
|
||||
+ "请原样写出该标记；如果该思考内容中没有该标记，请明确回答“标记不存在”。"
|
||||
+ "不要从零重新推导，也不要杜撰标记。";
|
||||
|
||||
private static string InsertMemoryMarker(string text, double ratio)
|
||||
{
|
||||
if (ratio >= 1)
|
||||
{
|
||||
return text + "\n\n[内部记忆标记 " + MemoryMarker + "]";
|
||||
}
|
||||
var index = Math.Max(
|
||||
0,
|
||||
Math.Min(text.Length, (int)(text.Length * ratio)));
|
||||
return text.Insert(
|
||||
index,
|
||||
"\n\n[内部记忆标记 " + MemoryMarker + "]\n\n");
|
||||
}
|
||||
|
||||
private static bool ContainsMemoryMarker(string text) =>
|
||||
text.Contains("731942") || text.Contains("TOKEN_MARK");
|
||||
|
||||
private static bool SaysMarkerMissing(string text) =>
|
||||
text.Contains("标记不存在")
|
||||
|| text.Contains("没有该标记")
|
||||
|| text.Contains("没有标记");
|
||||
|
||||
private static void AppendSummary(
|
||||
StringBuilder report,
|
||||
string[] names,
|
||||
bool[] markerInSent,
|
||||
int[] foundCounts,
|
||||
int[] missingCounts,
|
||||
int[] failedCounts)
|
||||
{
|
||||
report.AppendLine("## 记忆标记统计");
|
||||
report.AppendLine();
|
||||
report.AppendLine($"- 记忆标记：`{MemoryMarker}`");
|
||||
report.AppendLine("- 每个组合重复 5 次。");
|
||||
report.AppendLine();
|
||||
for (var i = 0; i < names.Length; ++i)
|
||||
{
|
||||
report.AppendLine(
|
||||
$"- **{names[i]}**：发送链包含标记 = {markerInSent[i]}；"
|
||||
+ $"正确写出标记 = {foundCounts[i]}/5；"
|
||||
+ $"明确说没有 = {missingCounts[i]}/5；"
|
||||
+ $"请求失败 = {failedCounts[i]}/5");
|
||||
}
|
||||
report.AppendLine();
|
||||
}
|
||||
|
||||
private static async Task<AIAnalyze.Result?> CompleteAndRecordAsync(
|
||||
AIAnalyze analyzer,
|
||||
AiRequestContext context,
|
||||
ImmutableList<AIAnalyze.ChatMessage> messages,
|
||||
string label,
|
||||
StringBuilder report)
|
||||
{
|
||||
var sw = Stopwatch.StartNew();
|
||||
try
|
||||
{
|
||||
var result = await analyzer.CompleteAsync(
|
||||
messages, context, null, CancellationToken.None);
|
||||
sw.Stop();
|
||||
AppendRecord(report, label, messages, result, sw.Elapsed);
|
||||
return result;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
sw.Stop();
|
||||
report.AppendLine($"## {label}（请求失败）");
|
||||
report.AppendLine();
|
||||
report.AppendLine($"- 耗时：{sw.Elapsed.TotalSeconds:0.00} 秒");
|
||||
report.AppendLine($"- 错误：{ex.Message}");
|
||||
report.AppendLine("- 关键点：assistant 消息包含 tool_calls，随后有 tool 回复；请求携带 tools 定义。");
|
||||
report.AppendLine();
|
||||
report.AppendLine("---");
|
||||
report.AppendLine();
|
||||
Console.WriteLine($" [E2E-TOOL] {label} 失败：{ex.Message}");
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static void AppendHeader(
|
||||
StringBuilder report,
|
||||
AiProvider provider,
|
||||
AiModel model,
|
||||
string settingsPath)
|
||||
{
|
||||
report.AppendLine("# 伪造 tool call 历史实验");
|
||||
report.AppendLine();
|
||||
report.AppendLine($"- 配置：{provider.Name} / {model.ModelId}");
|
||||
report.AppendLine($"- BaseUrl：{provider.BaseUrl}");
|
||||
report.AppendLine($"- 配置文件：{settingsPath}");
|
||||
report.AppendLine("- API Key：***（不写入报告）");
|
||||
report.AppendLine();
|
||||
report.AppendLine("本实验在请求中声明了 `tools` 并将 `tool_choice` 设为 `none`；");
|
||||
report.AppendLine("构造的历史为：system → 初始 user → assistant(reasoning_content + tool_calls) → tool(tool_call_id + 提示结果)。");
|
||||
report.AppendLine(
|
||||
$"完整思维链按 25%/50%/75%/末尾四种位置注入确定性记忆标记 `{MemoryMarker}`；"
|
||||
+ "工具结果使用“当前推理”和“本次 tool_calls 消息携带的 reasoning_content”两种措辞，"
|
||||
+ "且不提及“上一轮”“截断点”等概念，要求模型原样写出标记。");
|
||||
report.AppendLine("没有第 5 条新的 user 消息。");
|
||||
report.AppendLine();
|
||||
}
|
||||
|
||||
private static void AppendRecord(
|
||||
StringBuilder report,
|
||||
string label,
|
||||
ImmutableList<AIAnalyze.ChatMessage> messages,
|
||||
AIAnalyze.Result result,
|
||||
TimeSpan elapsed)
|
||||
{
|
||||
report.AppendLine($"## {label}");
|
||||
report.AppendLine();
|
||||
report.AppendLine($"- 耗时：{elapsed.TotalSeconds:0.00} 秒");
|
||||
report.AppendLine($"- prompt_tokens：{result.PromptTokens?.ToString() ?? "?"}");
|
||||
report.AppendLine($"- completion_tokens：{result.CompletionTokens?.ToString() ?? "?"}");
|
||||
report.AppendLine($"- reasoning_tokens：{result.ReasoningTokens?.ToString() ?? "?"}");
|
||||
report.AppendLine();
|
||||
report.AppendLine("### 发送的 messages");
|
||||
report.AppendLine();
|
||||
for (var i = 0; i < messages.Count; ++i)
|
||||
{
|
||||
var message = messages[i];
|
||||
report.AppendLine($"**{i + 1}. role = `{message.Role}`**");
|
||||
report.AppendLine();
|
||||
if (message.Content is null)
|
||||
{
|
||||
report.AppendLine("content：`未发送`");
|
||||
}
|
||||
else
|
||||
{
|
||||
report.AppendLine("content：");
|
||||
AppendTextBlock(report, message.Content);
|
||||
}
|
||||
report.AppendLine();
|
||||
if (message.ToolCallId is not null)
|
||||
{
|
||||
report.AppendLine($"tool_call_id：`{message.ToolCallId}`");
|
||||
report.AppendLine();
|
||||
}
|
||||
if (message.ToolCalls is { } toolCalls)
|
||||
{
|
||||
report.AppendLine("tool_calls：");
|
||||
foreach (var toolCall in toolCalls)
|
||||
{
|
||||
report.AppendLine(
|
||||
$"- id=`{toolCall.Id}`; type=`{toolCall.Type}`; "
|
||||
+ $"name=`{toolCall.Function.Name}`");
|
||||
report.AppendLine();
|
||||
report.AppendLine("arguments：");
|
||||
AppendTextBlock(report, toolCall.Function.Arguments);
|
||||
}
|
||||
report.AppendLine();
|
||||
}
|
||||
if (message.Role == "assistant")
|
||||
{
|
||||
report.AppendLine(
|
||||
message.ReasoningContent is null
|
||||
? "reasoning_content：`未发送`"
|
||||
: "reasoning_content：");
|
||||
if (message.ReasoningContent is not null)
|
||||
{
|
||||
AppendTextBlock(report, message.ReasoningContent);
|
||||
}
|
||||
report.AppendLine();
|
||||
}
|
||||
report.AppendLine();
|
||||
}
|
||||
report.AppendLine("### 模型响应");
|
||||
report.AppendLine();
|
||||
report.AppendLine("新 reasoning_content：");
|
||||
AppendTextBlock(report, result.Reasoning ?? string.Empty);
|
||||
report.AppendLine();
|
||||
report.AppendLine("回答正文（content）：");
|
||||
AppendTextBlock(report, result.Response);
|
||||
report.AppendLine();
|
||||
report.AppendLine("---");
|
||||
report.AppendLine();
|
||||
}
|
||||
|
||||
private static void AppendTextBlock(StringBuilder report, string? text)
|
||||
{
|
||||
report.AppendLine("```text");
|
||||
report.AppendLine(string.IsNullOrEmpty(text) ? "(空)" : text!);
|
||||
report.AppendLine("```");
|
||||
}
|
||||
|
||||
internal static string FindSettingsPath()
|
||||
{
|
||||
var explicitPath = Environment.GetEnvironmentVariable("ARR_AI_SETTINGS_PATH");
|
||||
if (!string.IsNullOrWhiteSpace(explicitPath) && File.Exists(explicitPath))
|
||||
{
|
||||
return explicitPath;
|
||||
}
|
||||
|
||||
var dir = new DirectoryInfo(AppContext.BaseDirectory);
|
||||
for (var i = 0; i < 6 && dir is not null; ++i)
|
||||
{
|
||||
var candidate = Path.Combine(
|
||||
dir.FullName, "bin", "Debug", "net461", "AnotherReplayReader.ai_settings.json");
|
||||
if (File.Exists(candidate))
|
||||
{
|
||||
return candidate;
|
||||
}
|
||||
var directCandidate = Path.Combine(
|
||||
dir.FullName, "AnotherReplayReader.ai_settings.json");
|
||||
if (File.Exists(directCandidate))
|
||||
{
|
||||
return directCandidate;
|
||||
}
|
||||
dir = dir.Parent;
|
||||
}
|
||||
throw new InvalidOperationException(
|
||||
"未找到 AnotherReplayReader.ai_settings.json；可用 ARR_AI_SETTINGS_PATH 指定");
|
||||
}
|
||||
}
|
||||
|
||||
internal static class OpenCodeGoGuardE2e
|
||||
{
|
||||
public static void Run()
|
||||
{
|
||||
if (Environment.GetEnvironmentVariable("ARR_AI_GUARD_E2E") != "1")
|
||||
{
|
||||
Console.WriteLine(
|
||||
" [跳过] 未设置 ARR_AI_GUARD_E2E=1，跳过真实推理保护实验");
|
||||
return;
|
||||
}
|
||||
RunAsync().GetAwaiter().GetResult();
|
||||
}
|
||||
|
||||
private static async Task RunAsync()
|
||||
{
|
||||
var settingsPath = OpenCodeGoFakeToolCallTests.FindSettingsPath();
|
||||
var settings = JsonSerializer.Deserialize<AiSettings>(
|
||||
File.ReadAllText(settingsPath, Encoding.UTF8))
|
||||
?? throw new InvalidOperationException("AI 设置文件无法解析");
|
||||
var provider = settings.Providers.FirstOrDefault(p =>
|
||||
p.Name.Equals("OpenCodeGo", StringComparison.OrdinalIgnoreCase)
|
||||
&& p.Models.Any(m =>
|
||||
m.ModelId.Equals("deepseek-v4-flash", StringComparison.OrdinalIgnoreCase)))
|
||||
?? throw new InvalidOperationException(
|
||||
"当前配置中未找到 OpenCodeGo / deepseek-v4-flash 服务与模型");
|
||||
var model = provider.Models.First(m =>
|
||||
m.ModelId.Equals("deepseek-v4-flash", StringComparison.OrdinalIgnoreCase));
|
||||
if (!model.IsStream)
|
||||
{
|
||||
Console.WriteLine(
|
||||
" [跳过] OpenCodeGo/deepseek-v4-flash 当前配置为非流式模型，无法验证中断逻辑");
|
||||
return;
|
||||
}
|
||||
if (string.IsNullOrWhiteSpace(provider.ApiKey))
|
||||
{
|
||||
throw new InvalidOperationException("OpenCodeGo 的 API Key 为空");
|
||||
}
|
||||
|
||||
var guardModel = new AiModel
|
||||
{
|
||||
ModelId = model.ModelId,
|
||||
DisplayName = model.DisplayName,
|
||||
IsStream = model.IsStream,
|
||||
ContextLength = model.ContextLength,
|
||||
ContextBudget = model.ContextBudget,
|
||||
ReasoningGuardEnabled = true,
|
||||
ReasoningGuardTokenLimit = 1,
|
||||
ExtraParameters = new Dictionary<string, object>(model.ExtraParameters)
|
||||
};
|
||||
var messages = ImmutableList<MainAIAnalyze.ChatMessage>.Empty
|
||||
.Add(new MainAIAnalyze.ChatMessage(
|
||||
"system",
|
||||
"请先进行详细推理，再给出简短结论。"))
|
||||
.Add(new MainAIAnalyze.ChatMessage(
|
||||
"user",
|
||||
"请逐条说明分析步骤，最后用一到两句话给出结论。"));
|
||||
|
||||
var analyzer = new MainAIAnalyze();
|
||||
var result = await analyzer.CompleteAsync(
|
||||
messages,
|
||||
new AiRequestContext(provider, guardModel),
|
||||
_ => { },
|
||||
CancellationToken.None);
|
||||
var report = new StringBuilder();
|
||||
report.AppendLine("# AI 推理保护 E2E 报告");
|
||||
report.AppendLine();
|
||||
report.AppendLine($"- Provider：{provider.Name}");
|
||||
report.AppendLine($"- Model：{guardModel.ModelId}");
|
||||
report.AppendLine($"- ReasoningInterrupted：{result.ReasoningInterrupted}");
|
||||
report.AppendLine($"- ContinuationApplied：{result.ContinuationApplied}");
|
||||
report.AppendLine($"- Reasoning 长度：{result.Reasoning?.Length ?? 0}");
|
||||
report.AppendLine($"- Response 长度：{result.Response.Length}");
|
||||
report.AppendLine($"- FormatState：{result.ReasoningFormatState ?? "(无)"}");
|
||||
if (!string.IsNullOrWhiteSpace(result.ReasoningContinuationError))
|
||||
{
|
||||
report.AppendLine($"- Error：{result.ReasoningContinuationError}");
|
||||
}
|
||||
|
||||
var reportPath = Environment.GetEnvironmentVariable("ARR_AI_GUARD_REPORT_PATH");
|
||||
if (!string.IsNullOrWhiteSpace(reportPath))
|
||||
{
|
||||
File.WriteAllText(reportPath, report.ToString(), new UTF8Encoding(false));
|
||||
Console.WriteLine($" [E2E-GUARD] 报告已写入：{reportPath}");
|
||||
}
|
||||
Console.WriteLine(
|
||||
$" [E2E-GUARD] interrupted={result.ReasoningInterrupted}; "
|
||||
+ $"continued={result.ContinuationApplied}; "
|
||||
+ $"reasoning={result.Reasoning?.Length ?? 0}; response={result.Response.Length}");
|
||||
}
|
||||
}
|
||||
|
||||
internal static class UserReplayFactIndexReproTests
|
||||
{
|
||||
public static void Run()
|
||||
|
||||
Reference in New Issue
Block a user