wip 2
This commit is contained in:
+61
-4
@@ -51,14 +51,71 @@ namespace AnotherReplayReader.Utils
|
||||
|
||||
if (slices.Count > MaxSlices)
|
||||
{
|
||||
var totalTokens = spans.Sum(s => s.EstimatedTokens);
|
||||
var raisedBudget = Math.Max(budget, (int)Math.Ceiling(totalTokens / (double)MaxSlices));
|
||||
slices = SliceCore(fullText, spans, raisedBudget, Math.Max(overlap, raisedBudget / 12));
|
||||
warnings.Add($"分段数超过上限 {MaxSlices},已放宽单段预算到 {raisedBudget:N0} token。");
|
||||
// 先尝试压缩噪声事件块(纯选择/编队类),避免一超限就放宽预算。
|
||||
var compressedSpans = CompressNoiseSpans(fullText, spans);
|
||||
if (compressedSpans.Length < spans.Length && !compressedSpans.IsEmpty)
|
||||
{
|
||||
var compressedSlices = SliceCore(fullText, compressedSpans, budget, overlap);
|
||||
if (compressedSlices.Count <= MaxSlices)
|
||||
{
|
||||
warnings.Add($"分段数超过上限 {MaxSlices},已过滤纯选择/编队事件块,压缩到 {compressedSlices.Count} 段。");
|
||||
return (compressedSlices.ToImmutableArray(), warnings.ToImmutable());
|
||||
}
|
||||
|
||||
var compressedTokens = compressedSpans.Sum(s => s.EstimatedTokens);
|
||||
var raisedBudget = Math.Max(budget, (int)Math.Ceiling(compressedTokens / (double)MaxSlices));
|
||||
slices = SliceCore(fullText, compressedSpans, raisedBudget, Math.Max(overlap, raisedBudget / 12));
|
||||
warnings.Add($"分段数仍超过上限 {MaxSlices},已过滤噪声并放宽单段预算到 {raisedBudget:N0} token。");
|
||||
}
|
||||
else
|
||||
{
|
||||
var totalTokens = spans.Sum(s => s.EstimatedTokens);
|
||||
var raisedBudget = Math.Max(budget, (int)Math.Ceiling(totalTokens / (double)MaxSlices));
|
||||
slices = SliceCore(fullText, spans, raisedBudget, Math.Max(overlap, raisedBudget / 12));
|
||||
warnings.Add($"分段数超过上限 {MaxSlices},已放宽单段预算到 {raisedBudget:N0} token。");
|
||||
}
|
||||
}
|
||||
return (slices.ToImmutableArray(), warnings.ToImmutable());
|
||||
}
|
||||
|
||||
private static ImmutableArray<EventSpan> CompressNoiseSpans(
|
||||
string fullText,
|
||||
ImmutableArray<EventSpan> spans)
|
||||
{
|
||||
var result = ImmutableArray.CreateBuilder<EventSpan>();
|
||||
foreach (var span in spans)
|
||||
{
|
||||
if (!IsNoiseEventBlock(fullText.Substring(span.StartIndex, span.Length)))
|
||||
{
|
||||
result.Add(span);
|
||||
}
|
||||
}
|
||||
return result.ToImmutable();
|
||||
}
|
||||
|
||||
private static bool IsNoiseEventBlock(string text)
|
||||
{
|
||||
var hasCommand = false;
|
||||
foreach (var rawLine in text.Replace("\r", "").Split('\n'))
|
||||
{
|
||||
var line = rawLine.Trim();
|
||||
if (string.IsNullOrWhiteSpace(line) || line.StartsWith("[", StringComparison.Ordinal))
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
// 命令行通常形如 "PlayerA: 选择单位";测试/简写文本把非空行也视作命令。
|
||||
hasCommand = true;
|
||||
if (line.IndexOf("选择", StringComparison.Ordinal) < 0
|
||||
&& line.IndexOf("编队", StringComparison.Ordinal) < 0
|
||||
&& line.IndexOf("取消选择", StringComparison.Ordinal) < 0)
|
||||
{
|
||||
return false;
|
||||
}
|
||||
}
|
||||
return hasCommand;
|
||||
}
|
||||
|
||||
private static List<ReplaySlice> SliceCore(
|
||||
string fullText,
|
||||
ImmutableArray<EventSpan> spans,
|
||||
|
||||
Reference in New Issue
Block a user