From bcf8c3f2048ebb9fe7a986266340e0775fc00dfe Mon Sep 17 00:00:00 2001 From: Tianyu Li Date: Tue, 30 Nov 2021 21:08:00 -0500 Subject: [PATCH 1/2] fix warnings --- cs/src/core/Index/FasterLog/FasterLog.cs | 9 ++++++--- cs/src/core/Index/FasterLog/FasterLogCommitPolicies.cs | 5 ----- cs/src/core/Index/FasterLog/FasterLogRecoveryInfo.cs | 3 +++ cs/src/core/Index/FasterLog/FasterLogSettings.cs | 3 +++ 4 files changed, 12 insertions(+), 8 deletions(-) diff --git a/cs/src/core/Index/FasterLog/FasterLog.cs b/cs/src/core/Index/FasterLog/FasterLog.cs index 9ccddcf17..e594cdc30 100644 --- a/cs/src/core/Index/FasterLog/FasterLog.cs +++ b/cs/src/core/Index/FasterLog/FasterLog.cs @@ -17,7 +17,7 @@ namespace FASTER.core /// /// FASTER log /// - public sealed partial class FasterLog : IDisposable + public sealed partial class FasterLog { private readonly BlittableAllocator allocator; private readonly LightEpoch epoch; @@ -519,6 +519,7 @@ public void Commit(bool spinWait = false) /// Proposal for the identifier to use for this commit, or -1 if the system should pick one. If supplied with /// a non -1 value, commit is guaranteed to have the supplied identifier if commit call is successful /// + /// callback function that will be invoked when strong commit is persistent /// Whether commit is successful public bool CommitStrongly(out long commitTail, out long actualCommitNum, bool spinWait = false, byte[] cookie = null, long proposedCommitNum = -1, Action callback = null) { @@ -1072,7 +1073,7 @@ private void UpdateCommittedState(FasterLogRecoveryInfo recoveryInfo) private void WriteCommitMetadata(FasterLogRecoveryInfo recoveryInfo) { - // TODO(Tianyu): If fast commit, write this in separate thread? + // TODO: can change to write this in separate thread for fast commit logCommitManager.Commit(recoveryInfo.BeginAddress, recoveryInfo.UntilAddress, recoveryInfo.ToByteArray(), recoveryInfo.CommitNum); // If not fast committing, set committed state as we commit metadata explicitly only after metadata commit @@ -1138,7 +1139,7 @@ private void SerialCommitCallbackWorker(CommitInfo commitInfo) if (latestCommit.FastForwardAllowed) WriteCommitMetadata(latestCommit); } - // TODO(Tianyu): Can invoke earlier in the case of fast commit + // TODO: Can invoke earlier in the case of fast commit var _commitTcs = commitTcs; commitTcs = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); var lci = new LinkedCommitInfo @@ -1717,6 +1718,8 @@ private bool CommitInternal(out long commitTail, out long actualCommitNum, bool info.UntilAddress = commitTail = TailAddress; } + Utility.MonotonicUpdate(ref commitCoveredAddress, commitTail, out _); + commitPolicy.OnCommitCreated(info); // Enqueue the commit record's content and offset into the queue so it can be picked up by the next flush // At this point, we expect the commit record to be flushed out as a distinct recovery point diff --git a/cs/src/core/Index/FasterLog/FasterLogCommitPolicies.cs b/cs/src/core/Index/FasterLog/FasterLogCommitPolicies.cs index 2e6218e86..72936e0f6 100644 --- a/cs/src/core/Index/FasterLog/FasterLogCommitPolicies.cs +++ b/cs/src/core/Index/FasterLog/FasterLogCommitPolicies.cs @@ -182,11 +182,6 @@ public sealed partial class FasterLog : IDisposable /// automatically retried. /// /// maximum number of commits that can be outstanding at a time - /// - /// whether to automatically retry rejected commit requests later. If set to true, even when - /// a commit() returns false due to being limited, the tail as of that commit will eventually be committed - /// without the need to invoke commit() again. - /// /// policy object public static IFasterLogCommitPolicy MaxParallelCommitStrategy(int k) => new MaxParallelCommitPolicy(k); diff --git a/cs/src/core/Index/FasterLog/FasterLogRecoveryInfo.cs b/cs/src/core/Index/FasterLog/FasterLogRecoveryInfo.cs index a3ec19d42..38674bce4 100644 --- a/cs/src/core/Index/FasterLog/FasterLogRecoveryInfo.cs +++ b/cs/src/core/Index/FasterLog/FasterLogRecoveryInfo.cs @@ -197,6 +197,9 @@ public readonly byte[] ToByteArray() return ms.ToArray(); } + /// + /// + /// size of this recovery info serialized public int SerializedSize() { var iteratorSize = sizeof(int); diff --git a/cs/src/core/Index/FasterLog/FasterLogSettings.cs b/cs/src/core/Index/FasterLog/FasterLogSettings.cs index c36cf0715..16d239b04 100644 --- a/cs/src/core/Index/FasterLog/FasterLogSettings.cs +++ b/cs/src/core/Index/FasterLog/FasterLogSettings.cs @@ -102,6 +102,9 @@ public class FasterLogSettings /// public bool RemoveOutdatedCommitFiles = true; + /// + /// CommitPolicy that influences the behavior of Commit() calls, or null if default. + /// public IFasterLogCommitPolicy CommitPolicy = null; internal LogSettings GetLogSettings() From 97c85d8211d200c0eeeae6ccf76031589f58e64e Mon Sep 17 00:00:00 2001 From: Tianyu Li Date: Tue, 30 Nov 2021 21:33:27 -0500 Subject: [PATCH 2/2] fix broken test --- cs/src/core/Index/FasterLog/FasterLog.cs | 4 ++-- cs/test/LogShiftTailStressTest.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cs/src/core/Index/FasterLog/FasterLog.cs b/cs/src/core/Index/FasterLog/FasterLog.cs index e594cdc30..87a615658 100644 --- a/cs/src/core/Index/FasterLog/FasterLog.cs +++ b/cs/src/core/Index/FasterLog/FasterLog.cs @@ -1680,12 +1680,12 @@ private bool CommitInternal(out long commitTail, out long actualCommitNum, bool info.SnapshotIterators(PersistedIterators); var metadataChanged = ShouldCommmitMetadata(ref info); // Only apply commit policy if not a strong commit - if (!fastForwardAllowed && !commitPolicy.AdmitCommit(TailAddress, metadataChanged)) + if (fastForwardAllowed && !commitPolicy.AdmitCommit(TailAddress, metadataChanged)) return false; lock (ongoingCommitRequests) { - if (commitCoveredAddress == TailAddress && metadataChanged) + if (commitCoveredAddress == TailAddress && !metadataChanged) // Nothing to commit if no metadata update and no new entries return false; // Make sure we will not be allowed to back out of a commit of AdmitCommit returns true, as the strategy diff --git a/cs/test/LogShiftTailStressTest.cs b/cs/test/LogShiftTailStressTest.cs index 503ffb0f9..12a926c0a 100644 --- a/cs/test/LogShiftTailStressTest.cs +++ b/cs/test/LogShiftTailStressTest.cs @@ -21,7 +21,7 @@ public void FasterLogShiftTailStressTest() { // Get an excruciatingly slow storage device to maximize chance of clogging the flush pipeline device = new LocalMemoryDevice(1L << 28, 1 << 28, 2, sector_size: 512, latencyMs: 50, fileName: "stress.log"); - var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = LogChecksumType.None, LogCommitManager = manager}; + var logSettings = new FasterLogSettings { LogDevice = device, LogChecksum = LogChecksumType.None, LogCommitManager = manager, SegmentSizeBits = 28}; log = new FasterLog(logSettings); byte[] entry = new byte[entryLength];