This commit is contained in:
2026-08-22 00:59:46 +02:00
parent 9250528442
commit a2f0bcb371
6 changed files with 292 additions and 6 deletions
+61 -4
View File
@@ -51,14 +51,71 @@ namespace AnotherReplayReader.Utils
if (slices.Count > MaxSlices)
{
var totalTokens = spans.Sum(s => s.EstimatedTokens);
var raisedBudget = Math.Max(budget, (int)Math.Ceiling(totalTokens / (double)MaxSlices));
slices = SliceCore(fullText, spans, raisedBudget, Math.Max(overlap, raisedBudget / 12));
warnings.Add($"分段数超过上限 {MaxSlices},已放宽单段预算到 {raisedBudget:N0} token。");
// 先尝试压缩噪声事件块(纯选择/编队类),避免一超限就放宽预算。
var compressedSpans = CompressNoiseSpans(fullText, spans);
if (compressedSpans.Length < spans.Length && !compressedSpans.IsEmpty)
{
var compressedSlices = SliceCore(fullText, compressedSpans, budget, overlap);
if (compressedSlices.Count <= MaxSlices)
{
warnings.Add($"分段数超过上限 {MaxSlices},已过滤纯选择/编队事件块,压缩到 {compressedSlices.Count} 段。");
return (compressedSlices.ToImmutableArray(), warnings.ToImmutable());
}
var compressedTokens = compressedSpans.Sum(s => s.EstimatedTokens);
var raisedBudget = Math.Max(budget, (int)Math.Ceiling(compressedTokens / (double)MaxSlices));
slices = SliceCore(fullText, compressedSpans, raisedBudget, Math.Max(overlap, raisedBudget / 12));
warnings.Add($"分段数仍超过上限 {MaxSlices},已过滤噪声并放宽单段预算到 {raisedBudget:N0} token。");
}
else
{
var totalTokens = spans.Sum(s => s.EstimatedTokens);
var raisedBudget = Math.Max(budget, (int)Math.Ceiling(totalTokens / (double)MaxSlices));
slices = SliceCore(fullText, spans, raisedBudget, Math.Max(overlap, raisedBudget / 12));
warnings.Add($"分段数超过上限 {MaxSlices},已放宽单段预算到 {raisedBudget:N0} token。");
}
}
return (slices.ToImmutableArray(), warnings.ToImmutable());
}
private static ImmutableArray<EventSpan> CompressNoiseSpans(
string fullText,
ImmutableArray<EventSpan> spans)
{
var result = ImmutableArray.CreateBuilder<EventSpan>();
foreach (var span in spans)
{
if (!IsNoiseEventBlock(fullText.Substring(span.StartIndex, span.Length)))
{
result.Add(span);
}
}
return result.ToImmutable();
}
private static bool IsNoiseEventBlock(string text)
{
var hasCommand = false;
foreach (var rawLine in text.Replace("\r", "").Split('\n'))
{
var line = rawLine.Trim();
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("[", StringComparison.Ordinal))
{
continue;
}
// 命令行通常形如 "PlayerA: 选择单位";测试/简写文本把非空行也视作命令。
hasCommand = true;
if (line.IndexOf("选择", StringComparison.Ordinal) < 0
&& line.IndexOf("编队", StringComparison.Ordinal) < 0
&& line.IndexOf("取消选择", StringComparison.Ordinal) < 0)
{
return false;
}
}
return hasCommand;
}
private static List<ReplaySlice> SliceCore(
string fullText,
ImmutableArray<EventSpan> spans,