diff --git a/docs/comparer.md b/docs/comparer.md index 69bcec06f..cf13dc690 100644 --- a/docs/comparer.md +++ b/docs/comparer.md @@ -42,6 +42,35 @@ The returned `CompareResult.NotEqual` takes an optional message that will be ren **If an input is split into multiple files, and a text file fails, then all subsequent binary comparisons will revert to the default comparison.** +### Bypass comparers for derived targets + +When a converter splits an input into multiple targets, for example a source document plus derived outputs such as rendered images or extracted text, a lenient comparer on a derived target can mask a real change in the source. Setting `BypassComparersForSubsequentOnDifference` on the source target ensures that, when the source differs from its verified file, all subsequent targets skip their registered comparers and fall back to exact comparison: + + + +```cs +// A converter that emits a canonical source document alongside derived targets (eg rendered pages). +// The source is flagged so that, when it differs, the derived targets skip their (potentially lenient) +// comparers and fall back to exact comparison, ensuring a real change in the source is never masked. +public static ConversionResult ConvertDocument(Stream document, IReadOnlyDictionary context) +{ + Target[] targets = + [ + new("docx", document) + { + BypassComparersForSubsequentOnDifference = true + }, + new("png", RenderPage(document)) + ]; + return new(info: null, targets); +} +``` +snippet source | anchor + + +The flag must be set on the source target, and that target must precede the derived targets in the conversion result. + + ### Instance comparer diff --git a/docs/mdsource/comparer.source.md b/docs/mdsource/comparer.source.md index cbadffb68..6dd416659 100644 --- a/docs/mdsource/comparer.source.md +++ b/docs/mdsource/comparer.source.md @@ -16,6 +16,15 @@ The returned `CompareResult.NotEqual` takes an optional message that will be ren **If an input is split into multiple files, and a text file fails, then all subsequent binary comparisons will revert to the default comparison.** +### Bypass comparers for derived targets + +When a converter splits an input into multiple targets, for example a source document plus derived outputs such as rendered images or extracted text, a lenient comparer on a derived target can mask a real change in the source. Setting `BypassComparersForSubsequentOnDifference` on the source target ensures that, when the source differs from its verified file, all subsequent targets skip their registered comparers and fall back to exact comparison: + +snippet: BypassComparersForSubsequentOnDifference + +The flag must be set on the source target, and that target must precede the derived targets in the conversion result. + + ### Instance comparer snippet: InstanceComparer diff --git a/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.bypassderived b/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.bypassderived new file mode 100644 index 000000000..7b6abb9f1 --- /dev/null +++ b/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.bypassderived @@ -0,0 +1 @@ +derived-other \ No newline at end of file diff --git a/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.bypasssource b/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.bypasssource new file mode 100644 index 000000000..fc2d6b26a --- /dev/null +++ b/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.bypasssource @@ -0,0 +1 @@ +source-other \ No newline at end of file diff --git a/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.txt b/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.txt new file mode 100644 index 000000000..c296c2eef --- /dev/null +++ b/src/Verify.Tests/Comparer/BypassComparerTests.SourceDiffers_DerivedComparerBypassed.verified.txt @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.bypassderived b/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.bypassderived new file mode 100644 index 000000000..7b6abb9f1 --- /dev/null +++ b/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.bypassderived @@ -0,0 +1 @@ +derived-other \ No newline at end of file diff --git a/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.bypasssource b/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.bypasssource new file mode 100644 index 000000000..d9fe910b4 --- /dev/null +++ b/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.bypasssource @@ -0,0 +1 @@ +source-content \ No newline at end of file diff --git a/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.txt b/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.txt new file mode 100644 index 000000000..c296c2eef --- /dev/null +++ b/src/Verify.Tests/Comparer/BypassComparerTests.SourceEqual_DerivedComparerUsed.verified.txt @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/src/Verify.Tests/Comparer/BypassComparerTests.cs b/src/Verify.Tests/Comparer/BypassComparerTests.cs new file mode 100644 index 000000000..fed1fa28a --- /dev/null +++ b/src/Verify.Tests/Comparer/BypassComparerTests.cs @@ -0,0 +1,48 @@ +public class BypassComparerTests +{ + // Mimics a converter that emits a canonical source target (eg a document) alongside a derived + // target (eg a rendered image) whose comparer can mask a real difference in the source. + static Target[] BuildTargets() => + [ + new("bypasssource", new MemoryStream("source-content"u8.ToArray())) + { + BypassComparersForSubsequentOnDifference = true + }, + new("bypassderived", new MemoryStream("derived-content"u8.ToArray())) + ]; + + static int derivedCompareCount; + + // A comparer that masks every difference by always reporting equal. + static Task MaskingDerivedComparer(Stream received, Stream verified, IReadOnlyDictionary context) + { + derivedCompareCount++; + return Task.FromResult(CompareResult.Equal); + } + + [Fact] + public async Task SourceEqual_DerivedComparerUsed() + { + derivedCompareCount = 0; + // The source matches its verified file, so no bypass is triggered and the derived comparer + // runs and masks the difference in the derived target. + await Verify(null, BuildTargets()) + .UseStreamComparer(MaskingDerivedComparer, "bypassderived") + .DisableDiff(); + Assert.Equal(1, derivedCompareCount); + } + + [Fact] + public async Task SourceDiffers_DerivedComparerBypassed() + { + derivedCompareCount = 0; + // The source differs from its verified file. Because it is flagged, the derived comparer is + // bypassed (never invoked) and the otherwise-masked derived difference is surfaced. + var exception = await Assert.ThrowsAsync( + () => Verify(null, BuildTargets()) + .UseStreamComparer(MaskingDerivedComparer, "bypassderived") + .DisableDiff()); + Assert.Equal(0, derivedCompareCount); + Assert.Contains("bypassderived", exception.Message); + } +} diff --git a/src/Verify.Tests/Snippets/BypassComparerSnippets.cs b/src/Verify.Tests/Snippets/BypassComparerSnippets.cs new file mode 100644 index 000000000..9f69c64dd --- /dev/null +++ b/src/Verify.Tests/Snippets/BypassComparerSnippets.cs @@ -0,0 +1,29 @@ +#if DEBUG + +public class BypassComparerSnippets +{ + #region BypassComparersForSubsequentOnDifference + + // A converter that emits a canonical source document alongside derived targets (eg rendered pages). + // The source is flagged so that, when it differs, the derived targets skip their (potentially lenient) + // comparers and fall back to exact comparison, ensuring a real change in the source is never masked. + public static ConversionResult ConvertDocument(Stream document, IReadOnlyDictionary context) + { + Target[] targets = + [ + new("docx", document) + { + BypassComparersForSubsequentOnDifference = true + }, + new("png", RenderPage(document)) + ]; + return new(info: null, targets); + } + + #endregion + + static Stream RenderPage(Stream document) => + new MemoryStream(); +} + +#endif diff --git a/src/Verify/Compare/Comparer.cs b/src/Verify/Compare/Comparer.cs index 54e56e1ae..9dde538f5 100644 --- a/src/Verify/Compare/Comparer.cs +++ b/src/Verify/Compare/Comparer.cs @@ -1,6 +1,6 @@ static class Comparer { - public static async Task Text(FilePair filePair, StringBuilder received, VerifySettings settings) + public static async Task Text(FilePair filePair, StringBuilder received, VerifySettings settings, bool bypassComparer = false) { IoHelpers.DeleteFileIfEmpty(filePair.VerifiedPath); if (!File.Exists(filePair.VerifiedPath)) @@ -10,7 +10,7 @@ public static async Task Text(FilePair filePair, StringBuilder r } var verified = await IoHelpers.ReadStringBuilderWithFixedLines(filePair.VerifiedPath); - var result = await CompareStrings(filePair.Extension, received, verified, settings); + var result = await CompareStrings(filePair.Extension, received, verified, settings, bypassComparer); if (result.IsEqual) { return new(Equality.Equal, null, received, verified); @@ -20,7 +20,7 @@ public static async Task Text(FilePair filePair, StringBuilder r return new(Equality.NotEqual, result.Message, received, verified); } - static Task CompareStrings(string extension, StringBuilder received, StringBuilder verified, VerifySettings settings) + static Task CompareStrings(string extension, StringBuilder received, StringBuilder verified, VerifySettings settings, bool bypassComparer) { if (verified.Length > 0 && verified.Length - 1 == received.Length && @@ -33,6 +33,7 @@ static Task CompareStrings(string extension, StringBuilder receiv #if NET6_0_OR_GREATER var isEqual = verified.Equals(received); if (!isEqual && + !bypassComparer && settings.TryFindStringComparer(extension, out var compare)) { return compare(received.ToString(), verified.ToString(), settings.Context); @@ -42,6 +43,7 @@ static Task CompareStrings(string extension, StringBuilder receiv var verifiedString = verified.ToString(); var isEqual = receivedString.Equals(verifiedString); if (!isEqual && + !bypassComparer && settings.TryFindStringComparer(extension, out var compare)) { return compare(receivedString, verifiedString, settings.Context); diff --git a/src/Verify/Compare/FileComparer.cs b/src/Verify/Compare/FileComparer.cs index 834be2433..5e04537b0 100644 --- a/src/Verify/Compare/FileComparer.cs +++ b/src/Verify/Compare/FileComparer.cs @@ -1,6 +1,6 @@ static class FileComparer { - public static async Task DoCompare(VerifySettings settings, FilePair file, bool previousTextFailed, Stream receivedStream) + public static async Task DoCompare(VerifySettings settings, FilePair file, bool bypassComparer, Stream receivedStream) { if (!File.Exists(file.VerifiedPath)) { @@ -14,7 +14,7 @@ public static async Task DoCompare(VerifySettings settings, File return new(Equality.NotEqual, null, null, null); } - if (!previousTextFailed && + if (!bypassComparer && settings.TryFindStreamComparer(file.Extension, out var compare)) { return await InnerCompare(file, receivedStream, compare, settings.Context); diff --git a/src/Verify/Target.cs b/src/Verify/Target.cs index dc0c9adf4..6ef58f27a 100644 --- a/src/Verify/Target.cs +++ b/src/Verify/Target.cs @@ -7,6 +7,16 @@ public readonly struct Target public string Extension { get; } public string? Name { get; } = null; public bool PerformConversion { get; } = true; + + /// + /// When true and this target differs from its verified file, all subsequent targets in the + /// same verification skip their registered comparers and fall back to exact (binary or string) comparison. + /// Intended for converters that emit a canonical source target (eg a document) alongside derived targets + /// (eg rendered images or extracted text) whose comparers may otherwise mask a real difference in the source. + /// Set this on the source target, and ensure it precedes the derived targets in the conversion result. + /// + public bool BypassComparersForSubsequentOnDifference { get; init; } + public string NameOrTarget => Name ?? "target"; public Stream StreamData diff --git a/src/Verify/Verifier/VerifyEngine.cs b/src/Verify/Verifier/VerifyEngine.cs index 35fadfbc0..bda78dc78 100644 --- a/src/Verify/Verifier/VerifyEngine.cs +++ b/src/Verify/Verifier/VerifyEngine.cs @@ -23,18 +23,18 @@ class VerifyEngine( public IReadOnlyList Equal => equal; public IReadOnlyList AutoVerified => autoVerified; - static async Task GetResult(VerifySettings settings, FilePair file, Target target, bool previousTextFailed) + static async Task GetResult(VerifySettings settings, FilePair file, Target target, bool textHasFailed, bool bypassComparers) { try { if (target.TryGetStringBuilder(out var value)) { - return await Comparer.Text(file, value, settings); + return await Comparer.Text(file, value, settings, bypassComparers); } using var stream = target.StreamData; stream.MoveToStart(); - return await FileComparer.DoCompare(settings, file, previousTextFailed, stream); + return await FileComparer.DoCompare(settings, file, textHasFailed || bypassComparers, stream); } catch (Exception exception) { @@ -54,21 +54,29 @@ public async Task HandleResults(List targetList) { var target = targetList[0]; var file = getFileNames(target); - var result = await GetResult(settings, file, target, false); + var result = await GetResult(settings, file, target, false, false); HandleCompareResult(result, file); return; } var textHasFailed = false; + var bypassComparers = false; async Task Inner(FilePair file, Target target) { - var result = await GetResult(settings, file, target, textHasFailed); + var result = await GetResult(settings, file, target, textHasFailed, bypassComparers); - if (file.IsText && - result.Equality != Equality.Equal) + if (result.Equality != Equality.Equal) { - textHasFailed = true; + if (file.IsText) + { + textHasFailed = true; + } + + if (target.BypassComparersForSubsequentOnDifference) + { + bypassComparers = true; + } } HandleCompareResult(result, file);