Files
AnotherReplayReader/Utils/AIAnalysisValidation.cs
T
2026-07-07 16:42:42 +02:00

677 lines
26 KiB
C#

using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Text.RegularExpressions;
namespace AnotherReplayReader.Utils
{
internal enum AIValidationSeverity
{
Info,
WeakEvidence,
Warning,
Contradiction,
Fatal
}
internal enum AIValidationIssueKind
{
InvalidMachineReadableClaims,
MissingMachineReadableClaims,
MissingAlternative,
WeakEvidence,
MissingEvidence,
InvalidEvidenceLevel,
UnitCapabilityContradiction,
UnitTimelineContradiction,
UnsupportedGameKnowledge
}
internal enum AIEvidenceLevel
{
Confirmed,
HighlyLikely,
Possible,
Uncertain,
RuledOut
}
internal sealed record AIUnitClaim(
string UnitId,
string Player,
string Claim,
AIEvidenceLevel EvidenceLevel,
ImmutableArray<string> Evidence,
ImmutableArray<string> Alternatives,
ImmutableArray<string> NeedsConfirmation);
internal sealed record AIEventClaim(
string Claim,
AIEvidenceLevel EvidenceLevel,
ImmutableArray<string> Evidence);
internal sealed record AITimelineClaim(
string Claim,
AIEvidenceLevel EvidenceLevel,
ImmutableArray<string> Evidence);
internal sealed record AIMachineReadableClaims(
ImmutableArray<AIUnitClaim> UnitClaims,
ImmutableArray<AIEventClaim> EventClaims,
ImmutableArray<AITimelineClaim> TimelineClaims)
{
public static AIMachineReadableClaims Empty { get; } = new(
ImmutableArray<AIUnitClaim>.Empty,
ImmutableArray<AIEventClaim>.Empty,
ImmutableArray<AITimelineClaim>.Empty);
}
internal sealed record AIValidationIssue(
AIValidationSeverity Severity,
AIValidationIssueKind Kind,
string Message,
string? UnitId = null,
TimeSpan? Time = null);
internal sealed record AIValidationResult(
AIMachineReadableClaims Claims,
ImmutableArray<AIValidationIssue> Issues)
{
public static AIValidationResult Empty { get; } =
new(AIMachineReadableClaims.Empty, ImmutableArray<AIValidationIssue>.Empty);
public bool RequiresRevision =>
Issues.Any(i => i.Severity is AIValidationSeverity.Contradiction or AIValidationSeverity.Fatal);
public bool HasIssues => !Issues.IsEmpty;
}
internal static class AIAnalysisValidation
{
private static readonly Regex _jsonFenceRegex = new(
@"```(?:json)?\s*(\{[\s\S]*?\})\s*```",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
public static AIValidationResult ValidateMachineReadableClaims(
string response,
ReplayFactIndex? factIndex = null)
{
var issues = ImmutableArray.CreateBuilder<AIValidationIssue>();
var json = ExtractJsonObject(response);
if (json is null || string.IsNullOrWhiteSpace(json))
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Warning,
AIValidationIssueKind.MissingMachineReadableClaims,
"AI 未输出机器可读声明,无法进行自动验证。"));
return new AIValidationResult(AIMachineReadableClaims.Empty, issues.ToImmutable());
}
var claims = ParseClaims(json, issues);
if (claims is null)
{
return new AIValidationResult(AIMachineReadableClaims.Empty, issues.ToImmutable());
}
ValidateClaimSelfConsistency(claims, issues);
ValidateUnpackAmbiguity(claims, issues);
if (factIndex is not null)
{
ValidateTimelineConsistency(claims, factIndex, issues);
}
return new AIValidationResult(claims, issues.ToImmutable());
}
public static string FormatIssues(ImmutableArray<AIValidationIssue> issues)
{
if (issues.IsEmpty)
{
return "未发现机器可读声明问题。";
}
var sb = new StringBuilder();
foreach (var issue in issues)
{
var unitText = string.IsNullOrWhiteSpace(issue.UnitId)
? string.Empty
: $" UnitId={issue.UnitId}";
sb.AppendLine($"[{issue.Severity}/{issue.Kind}]{unitText} {issue.Message}");
}
return sb.ToString().TrimEnd();
}
private static string? ExtractJsonObject(string response)
{
var markerIndex = response.LastIndexOf("[机器可读声明]", StringComparison.OrdinalIgnoreCase);
var searchText = markerIndex >= 0 ? response.Substring(markerIndex) : response;
var matches = _jsonFenceRegex.Matches(searchText);
if (matches.Count > 0)
{
return matches[matches.Count - 1].Groups[1].Value;
}
var start = searchText.LastIndexOf('{');
var end = searchText.LastIndexOf('}');
if (start >= 0 && end > start)
{
return searchText.Substring(start, end - start + 1);
}
return null;
}
private static AIMachineReadableClaims? ParseClaims(
string json,
ImmutableArray<AIValidationIssue>.Builder issues)
{
try
{
using var doc = JsonDocument.Parse(json, new JsonDocumentOptions
{
AllowTrailingCommas = true,
CommentHandling = JsonCommentHandling.Skip
});
var root = doc.RootElement;
if (root.ValueKind != JsonValueKind.Object)
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Warning,
AIValidationIssueKind.InvalidMachineReadableClaims,
"机器可读声明不是 JSON object。"));
return null;
}
return new AIMachineReadableClaims(
ReadUnitClaims(root, issues),
ReadSimpleClaims(root, "eventClaims", issues)
.Select(c => new AIEventClaim(c.Claim, c.EvidenceLevel, c.Evidence))
.ToImmutableArray(),
ReadSimpleClaims(root, "timelineClaims", issues)
.Select(c => new AITimelineClaim(c.Claim, c.EvidenceLevel, c.Evidence))
.ToImmutableArray());
}
catch (JsonException ex)
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Warning,
AIValidationIssueKind.InvalidMachineReadableClaims,
$"机器可读声明 JSON 解析失败:{ex.Message}"));
return null;
}
}
private const int MaxUnitClaims = 10;
private static ImmutableArray<AIUnitClaim> ReadUnitClaims(JsonElement root, ImmutableArray<AIValidationIssue>.Builder issues)
{
if (!TryGetArray(root, "unitClaims", out var unitClaims))
{
return ImmutableArray<AIUnitClaim>.Empty;
}
var result = ImmutableArray.CreateBuilder<AIUnitClaim>();
var totalCount = 0;
foreach (var item in unitClaims.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.Object)
{
continue;
}
totalCount++;
if (result.Count >= MaxUnitClaims)
{
continue;
}
result.Add(new AIUnitClaim(
ReadFlexibleString(item, "unitId"),
ReadFlexibleString(item, "player"),
ReadFlexibleString(item, "claim"),
ReadEvidenceLevel(item, issues),
ReadStringArray(item, "evidence"),
ReadStringArray(item, "alternatives"),
ReadStringArray(item, "needsConfirmation")));
}
if (totalCount > MaxUnitClaims)
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Info,
AIValidationIssueKind.InvalidMachineReadableClaims,
$"unitClaims 包含 {totalCount} 条声明,仅处理前 {MaxUnitClaims} 条,其余已忽略。"));
}
return result.ToImmutable();
}
private sealed record SimpleClaim(
string Claim,
AIEvidenceLevel EvidenceLevel,
ImmutableArray<string> Evidence);
private static ImmutableArray<SimpleClaim> ReadSimpleClaims(JsonElement root, string propertyName, ImmutableArray<AIValidationIssue>.Builder issues)
{
if (!TryGetArray(root, propertyName, out var claims))
{
return ImmutableArray<SimpleClaim>.Empty;
}
var result = ImmutableArray.CreateBuilder<SimpleClaim>();
var maxClaims = propertyName switch
{
"eventClaims" => 5,
"timelineClaims" => 3,
_ => 50,
};
var totalCount = 0;
foreach (var item in claims.EnumerateArray())
{
if (item.ValueKind != JsonValueKind.Object)
{
continue;
}
totalCount++;
if (result.Count >= maxClaims)
{
continue;
}
var claim = ReadFlexibleString(item, "claim");
if (string.IsNullOrWhiteSpace(claim) && propertyName == "eventClaims")
{
claim = ReadFlexibleString(item, "event");
}
result.Add(new SimpleClaim(
claim,
ReadEvidenceLevel(item, issues),
ReadStringArray(item, "evidence")));
}
if (totalCount > maxClaims)
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Info,
AIValidationIssueKind.InvalidMachineReadableClaims,
$"{propertyName} 包含 {totalCount} 条声明,仅处理前 {maxClaims} 条,其余已忽略。"));
}
return result.ToImmutable();
}
private static void ValidateClaimSelfConsistency(
AIMachineReadableClaims claims,
ImmutableArray<AIValidationIssue>.Builder issues)
{
foreach (var claim in claims.UnitClaims)
{
if (string.IsNullOrWhiteSpace(claim.UnitId))
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Warning,
AIValidationIssueKind.InvalidMachineReadableClaims,
"Unit claim 缺少 unitId。"));
}
if (string.IsNullOrWhiteSpace(claim.Claim))
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Warning,
AIValidationIssueKind.InvalidMachineReadableClaims,
"Unit claim 缺少 claim。",
claim.UnitId));
}
if (claim.EvidenceLevel is AIEvidenceLevel.Confirmed or AIEvidenceLevel.HighlyLikely
&& claim.Evidence.IsEmpty)
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Warning,
AIValidationIssueKind.MissingEvidence,
"高置信 UnitId 推测缺少 evidence。",
claim.UnitId));
}
if (claim.EvidenceLevel is AIEvidenceLevel.Possible or AIEvidenceLevel.Uncertain
&& claim.Alternatives.IsEmpty
&& claim.NeedsConfirmation.IsEmpty)
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.WeakEvidence,
AIValidationIssueKind.MissingAlternative,
"低置信 UnitId 推测应提供 alternatives 或 needsConfirmation。",
claim.UnitId));
}
}
}
#region Structured evidence parsing
internal enum AIEvidenceType
{
Build,
Place,
Produce,
Sell,
Select,
Move,
Power,
Unknown
}
internal sealed record StructuredEvidence(
AIEvidenceType Type,
string Time,
ImmutableArray<string> Parameters,
string Raw)
{
public string? GetSpecialPowerName() =>
Type == AIEvidenceType.Power && Parameters.Length >= 1 ? Parameters[0] : null;
public string? GetUnitId() => Type switch
{
AIEvidenceType.Build or AIEvidenceType.Place when Parameters.Length >= 2 => Parameters[1],
AIEvidenceType.Produce when Parameters.Length >= 2 => Parameters[1],
AIEvidenceType.Power when Parameters.Length >= 2 => Parameters[1],
AIEvidenceType.Sell when Parameters.Length >= 1 => Parameters[0],
AIEvidenceType.Select when Parameters.Length >= 1 => Parameters[0],
_ => null,
};
public string? GetAssetName() => Type switch
{
AIEvidenceType.Build or AIEvidenceType.Place or AIEvidenceType.Produce
when Parameters.Length >= 1 => Parameters[0],
_ => null,
};
}
private static readonly Regex _structuredEvidenceRegex = new(
@"^(build|place|produce|sell|select|move|power)\|([^|]+(?:\|(?!\|).*)?)$",
RegexOptions.IgnoreCase | RegexOptions.Compiled);
internal static StructuredEvidence ParseStructuredEvidence(string text)
{
var match = _structuredEvidenceRegex.Match(text.Trim());
if (!match.Success)
{
return new StructuredEvidence(AIEvidenceType.Unknown, string.Empty, ImmutableArray<string>.Empty, text);
}
var type = match.Groups[1].Value.ToLowerInvariant() switch
{
"build" => AIEvidenceType.Build,
"place" => AIEvidenceType.Place,
"produce" => AIEvidenceType.Produce,
"sell" => AIEvidenceType.Sell,
"select" => AIEvidenceType.Select,
"move" => AIEvidenceType.Move,
"power" => AIEvidenceType.Power,
_ => AIEvidenceType.Unknown,
};
var rest = match.Groups[2].Value;
var parts = rest.Split('|');
var time = parts.Length >= 1 ? parts[0].Trim() : string.Empty;
var parameters = parts.Skip(1).Select(p => p.Trim()).ToImmutableArray();
return new StructuredEvidence(type, time, parameters, text);
}
internal static ImmutableArray<StructuredEvidence> ParseAllEvidence(ImmutableArray<string> evidenceStrings)
{
return evidenceStrings
.Select(ParseStructuredEvidence)
.ToImmutableArray();
}
#endregion
#region Validation rules
private static void ValidateUnpackAmbiguity(
AIMachineReadableClaims claims,
ImmutableArray<AIValidationIssue>.Builder issues)
{
foreach (var claim in claims.UnitClaims)
{
if (claim.EvidenceLevel is not (AIEvidenceLevel.Confirmed or AIEvidenceLevel.HighlyLikely))
{
continue;
}
var evidence = ParseAllEvidence(claim.Evidence);
var hasUnpack = evidence.Any(e =>
e.GetSpecialPowerName() is string p &&
p.IndexOf("UnpackReplaceSelf", StringComparison.OrdinalIgnoreCase) >= 0);
if (!hasUnpack)
{
continue;
}
var hasPack = evidence.Any(e =>
e.GetSpecialPowerName() is string p &&
p.IndexOf("PackReplaceSelf", StringComparison.OrdinalIgnoreCase) >= 0);
if (hasPack)
{
continue;
}
issues.Add(new AIValidationIssue(
AIValidationSeverity.WeakEvidence,
AIValidationIssueKind.MissingAlternative,
$"UnitId {claim.UnitId} 使用了 UnpackReplaceSelf 但证据中无对应 PackReplaceSelf。UnpackReplaceSelf 可能对应基地车展开或矿车展开成指挥中心,建议降低置信度或添加 alternative。",
claim.UnitId));
}
}
private static void ValidateTimelineConsistency(
AIMachineReadableClaims claims,
ReplayFactIndex factIndex,
ImmutableArray<AIValidationIssue>.Builder issues)
{
foreach (var claim in claims.UnitClaims)
{
if (string.IsNullOrWhiteSpace(claim.UnitId))
{
continue;
}
if (!uint.TryParse(claim.UnitId, out var unitId))
{
continue;
}
// Check 1: UnitId referenced in claim exists in the replay
if (!factIndex.UnitIdFirstObservedTime.ContainsKey(unitId))
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Warning,
AIValidationIssueKind.InvalidMachineReadableClaims,
$"UnitId {claim.UnitId} 在回放数据中从未出现过,AI 可能编造了不存在的 UnitId。",
claim.UnitId));
continue;
}
// Check 2: Verify special power evidence against fact index
var evidence = ParseAllEvidence(claim.Evidence);
foreach (var ev in evidence)
{
if (ev.Type != AIEvidenceType.Power)
{
continue;
}
var evUnitIdStr = ev.GetUnitId();
if (evUnitIdStr is null || !uint.TryParse(evUnitIdStr, out var evUnitId))
{
continue;
}
var powerName = ev.GetSpecialPowerName();
if (powerName is null)
{
continue;
}
// Does this UnitId exist in the fact index?
if (!factIndex.UnitIdFirstObservedTime.ContainsKey(evUnitId))
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Info,
AIValidationIssueKind.InvalidMachineReadableClaims,
$"证据引用了回放中不存在的 UnitId {evUnitIdStr}。",
claim.UnitId));
continue;
}
// Did this UnitId actually use this special power?
if (factIndex.UnitIdSpecialPowers.TryGetValue(evUnitId, out var actualPowers)
&& !actualPowers.Contains(powerName))
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.Contradiction,
AIValidationIssueKind.UnitCapabilityContradiction,
$"UnitId {evUnitIdStr} 在回放中使用过以下特殊能力:{string.Join(", ", actualPowers.OrderBy(x => x))},但 AI 声称其使用了“{powerName}”——此能力未在该 UnitId 上观察到。",
claim.UnitId));
}
}
// Check 3: UnitId used as builder vs claim
var isBuilderInReplay = factIndex.BuilderUnitIds.Contains(unitId);
var claimLooksLikeBuilder = ClaimLooksLikeBuilder(claim.Claim);
if (claimLooksLikeBuilder && !isBuilderInReplay)
{
issues.Add(new AIValidationIssue(
AIValidationSeverity.WeakEvidence,
AIValidationIssueKind.UnitCapabilityContradiction,
$"AI 推测 UnitId {claim.UnitId} 是“{claim.Claim}”(推测是建造单位),但该 UnitId 在回放中从未作为建造者(建造建筑)出现。",
claim.UnitId));
}
}
}
#endregion
private static bool TryGetArray(JsonElement root, string propertyName, out JsonElement array)
{
if (root.TryGetProperty(propertyName, out array)
&& array.ValueKind == JsonValueKind.Array)
{
return true;
}
array = default;
return false;
}
private static AIEvidenceLevel ReadEvidenceLevel(JsonElement item, ImmutableArray<AIValidationIssue>.Builder issues)
{
var value = ReadFlexibleString(item, "evidenceLevel");
return NormalizeEvidenceLevel(value, issues);
}
private static AIEvidenceLevel NormalizeEvidenceLevel(string value, ImmutableArray<AIValidationIssue>.Builder issues)
{
value = value.Trim().Replace("_", "").Replace("-", "").Replace(" ", "");
var result = value.ToLowerInvariant() switch
{
"confirmed" or "确定" => AIEvidenceLevel.Confirmed,
"highlylikely" or "high" or "高度可能" => AIEvidenceLevel.HighlyLikely,
"possible" or "可能" => AIEvidenceLevel.Possible,
"ruledout" or "excluded" or "已排除" => AIEvidenceLevel.RuledOut,
_ => (AIEvidenceLevel?)null,
};
if (result is not null)
{
return result.Value;
}
issues.Add(new AIValidationIssue(
AIValidationSeverity.Info,
AIValidationIssueKind.InvalidEvidenceLevel,
$"无法识别的证据等级 \"{value.Trim()}\",已降级为不确定。"));
return AIEvidenceLevel.Uncertain;
}
private static string ReadFlexibleString(JsonElement item, string propertyName)
{
if (!item.TryGetProperty(propertyName, out var value))
{
return string.Empty;
}
return value.ValueKind switch
{
JsonValueKind.String => value.GetString() ?? string.Empty,
JsonValueKind.Number => value.GetRawText(),
JsonValueKind.True => "true",
JsonValueKind.False => "false",
_ => string.Empty,
};
}
private static ImmutableArray<string> ReadStringArray(JsonElement item, string propertyName)
{
if (!item.TryGetProperty(propertyName, out var value))
{
return ImmutableArray<string>.Empty;
}
if (value.ValueKind == JsonValueKind.String)
{
var text = value.GetString();
return string.IsNullOrWhiteSpace(text)
? ImmutableArray<string>.Empty
: ImmutableArray.Create(text!);
}
if (value.ValueKind != JsonValueKind.Array)
{
return ImmutableArray<string>.Empty;
}
var result = new List<string>();
foreach (var element in value.EnumerateArray())
{
var text = element.ValueKind == JsonValueKind.String
? element.GetString()
: element.GetRawText();
if (!string.IsNullOrWhiteSpace(text))
{
result.Add(text!);
}
}
return result.ToImmutableArray();
}
/// <summary>
/// Check if a claim text describes a builder unit (MCV, Nanocore, etc.).
/// Uses structured knowledge (knowledge_units.json) when available, falls
/// back to heuristic string matching for backward compatibility.
/// </summary>
private static bool ClaimLooksLikeBuilder(string claimText)
{
// Primary: structured knowledge lookup
var structured = StructuredKnowledge.Instance;
if (structured is not null)
{
foreach (var entity in structured.AllEntities)
{
if (entity.HasTag(KnowledgeTag.Builder) &&
claimText.IndexOf(entity.AssetName, StringComparison.OrdinalIgnoreCase) >= 0)
{
return true;
}
}
}
// Fallback: heuristic string matching
var keywords = new[] { "MCV", "基地车", "Nanocore", "纳米核心", "builder", "建造者" };
return keywords.Any(kw => claimText.IndexOf(kw, StringComparison.OrdinalIgnoreCase) >= 0);
}
}
}