diff --git a/backend/include/backend/session.hpp b/backend/include/backend/session.hpp index 299192cd..d9fd5725 100644 --- a/backend/include/backend/session.hpp +++ b/backend/include/backend/session.hpp @@ -189,7 +189,12 @@ class Session void registerRpcSftpAddBulkUploadOperation(); void registerRpcSftpAddBulkDeleteOperation(); void registerRpcSftpExistsBatch(); - void registerRpcSftpAddSyncScanOperation(); + void registerRpcSftpOpenSyncSession(); + void registerRpcSftpRecomputeSyncDiff(); + void registerRpcSftpLoadSyncDiffChildren(); + void registerRpcSftpBuildSyncEnqueuePlan(); + void registerRpcSftpCancelSyncDiff(); + void registerRpcSftpCloseSyncSession(); void registerOperationQueuePauseUnpause(); void removeChannel(Ids::ChannelId channelId); diff --git a/backend/include/backend/sftp/local_scan_operation.hpp b/backend/include/backend/sftp/local_scan_operation.hpp index 47d8e0b8..972de0fb 100644 --- a/backend/include/backend/sftp/local_scan_operation.hpp +++ b/backend/include/backend/sftp/local_scan_operation.hpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include @@ -26,6 +28,9 @@ class LocalScanOperation : public Operation bool recursive{true}; /// When true, entries whose filename starts with '.' are skipped during scan. bool ignoreHidden{false}; + /// When true, the operation builds a sorted @ref SharedData::Sync::ScanNode tree + /// instead of a flat entry vector. + bool buildTree{false}; }; LocalScanOperation(ScanOperationOptions options); @@ -81,10 +86,25 @@ class LocalScanOperation : public Operation return fn(static_cast(*walker_)); } + /** + * @brief Variant of @ref withWalkerDo that drives a sorted ScanNode tree walker. + * Only call when @ref ScanOperationOptions::buildTree was set. + */ + auto withTreeWalkerDo(auto&& fn) + { + auto scan = [this](std::filesystem::path const& path) + { + return scanner(path); + }; + using WalkerType = SharedData::Sync::TreeDirectoryWalker; + if (!walker_) + walker_ = std::make_unique(localPath_, std::move(scan)); + return fn(static_cast(*walker_)); + } + /** * @brief Eject the scanned directory entries. Careful!: The internal list is moved out. - * - * @return std::vector + * Only valid when @ref buildsTree() is false. */ std::vector ejectEntries() { @@ -96,6 +116,24 @@ class LocalScanOperation : public Operation ); } + /** + * @brief Eject the scanned node tree. Only valid when @ref buildsTree() is true. + */ + SharedData::Sync::ScanNode ejectScanTree() + { + return withTreeWalkerDo( + [](auto& walker) + { + return std::move(walker).ejectTree(); + } + ); + } + + bool buildsTree() const noexcept + { + return buildTree_; + } + std::uint64_t totalBytes() const; std::expected, Error> scanner(std::filesystem::path const& path); @@ -107,6 +145,7 @@ class LocalScanOperation : public Operation bool respectIgnoreFiles_; bool recursive_; bool ignoreHidden_; + bool buildTree_; bool rootScanned_{false}; SharedData::IgnoreMatcher ignoreMatcher_; std::unique_ptr walker_; diff --git a/backend/include/backend/sftp/operation_queue.hpp b/backend/include/backend/sftp/operation_queue.hpp index b56e5177..be87b948 100644 --- a/backend/include/backend/sftp/operation_queue.hpp +++ b/backend/include/backend/sftp/operation_queue.hpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -11,6 +12,10 @@ #include #include #include +#include +#include +#include +#include #include #include @@ -224,8 +229,14 @@ class OperationQueue * @param remotePath Remote directory root to scan. * @param localPath Local directory root to scan. */ + /** + * @brief Opens a @ref SyncSession and kicks off both scans. The scans build + * ScanNode trees directly and feed them into the session on completion. + * Emits @c onSyncScanPhaseDone(sessionId, isLocal) for each side. + */ void addSyncScanOperation( SecureShell::SftpSession& sftp, + Ids::SyncSessionId syncSessionId, Ids::OperationId remoteScanId, Ids::OperationId localScanId, std::filesystem::path const& remotePath, @@ -235,6 +246,20 @@ class OperationQueue bool ignoreHidden ); + /** + * @brief Returns the session with the given id, or nullptr when it has been + * closed / is unknown. Callers must post onto @ref SyncSession::strand() + * before touching mutable state. + */ + std::shared_ptr syncSession(Ids::SyncSessionId const& sessionId) const; + + /** + * @brief Removes the session from the registry. Flips its cancel flag first so + * any in-flight walk exits at its next checkpoint. The destructor runs + * once the last @c shared_ptr reference (captured tasks + this map) drops. + */ + void closeSyncSession(Ids::SyncSessionId const& sessionId); + void registerRpc(); bool paused() const; @@ -295,9 +320,15 @@ class OperationQueue std::deque>> operations_{}; std::atomic_bool paused_{true}; int parallelism_{1}; - // Keyed by operationId.value(). Called when a sync-only scan completes and ejects its results. - std::map, std::uint64_t)>> - syncScanCallbacks_{}; + // Keyed by operationId.value(). Called when a sync-only scan completes and + // hands off its ScanNode tree. The scan is known to be build-tree mode so + // @ref ScanOperation::ejectScanTree() has the payload. + std::map> syncScanCallbacks_{}; + + // SyncSessions active on this channel. Keyed by SyncSessionId.value() because + // Ids::IdHash isn't specialized on the strongly-typed id; the existing + // bulkResumeStash_ does the specialization — we stay string-keyed for simplicity. + std::map> syncSessions_{}; // Per-bulk backup snapshot, populated when a bulk-add path enqueues an // op and erased once the op completes (success or failure). Keyed by diff --git a/backend/include/backend/sftp/scan_operation.hpp b/backend/include/backend/sftp/scan_operation.hpp index e18e1e98..070ecce9 100644 --- a/backend/include/backend/sftp/scan_operation.hpp +++ b/backend/include/backend/sftp/scan_operation.hpp @@ -4,6 +4,8 @@ #include #include #include +#include +#include #include #include @@ -25,6 +27,10 @@ class ScanOperation : public Operation bool recursive{true}; /// When true, entries whose filename starts with '.' are skipped during scan. bool ignoreHidden{false}; + /// When true, the operation builds a sorted @ref SharedData::Sync::ScanNode tree + /// instead of a flat entry vector. Only one shape is active at a time; use + /// @ref ejectScanTree() in that case and @ref ejectEntries() otherwise. + bool buildTree{false}; }; SecureShell::ProcessingStrand* strand() const override; @@ -80,10 +86,25 @@ class ScanOperation : public Operation return fn(static_cast(*walker_)); } + /** + * @brief Variant of @ref withWalkerDo that drives a sorted ScanNode tree walker. + * Only call when @ref ScanOperationOptions::buildTree was set. + */ + auto withTreeWalkerDo(auto&& fn) + { + auto scan = [this](std::filesystem::path const& path) + { + return scanner(path); + }; + using WalkerType = SharedData::Sync::TreeDirectoryWalker; + if (!walker_) + walker_ = std::make_unique(remotePath_, std::move(scan)); + return fn(static_cast(*walker_)); + } + /** * @brief Eject the scanned directory entries. Careful!: The internal list is moved out. - * - * @return std::vector + * Only valid when @ref buildsTree() is false. */ std::vector ejectEntries() { @@ -95,6 +116,24 @@ class ScanOperation : public Operation ); } + /** + * @brief Eject the scanned node tree. Only valid when @ref buildsTree() is true. + */ + SharedData::Sync::ScanNode ejectScanTree() + { + return withTreeWalkerDo( + [](auto& walker) + { + return std::move(walker).ejectTree(); + } + ); + } + + bool buildsTree() const noexcept + { + return buildTree_; + } + std::uint64_t totalBytes() const; std::expected, Error> scanner(std::filesystem::path const& path); @@ -108,6 +147,7 @@ class ScanOperation : public Operation bool respectIgnoreFiles_; bool recursive_; bool ignoreHidden_; + bool buildTree_; bool rootScanned_{false}; SharedData::IgnoreMatcher ignoreMatcher_; std::unique_ptr walker_; diff --git a/backend/include/backend/sync/sync_session.hpp b/backend/include/backend/sync/sync_session.hpp new file mode 100644 index 00000000..5533f13a --- /dev/null +++ b/backend/include/backend/sync/sync_session.hpp @@ -0,0 +1,169 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/** + * @brief Backend-resident holder for a single sync flow. + * + * Lifecycle (driven from @ref OperationQueue RPC handlers): + * 1. Constructed when the frontend calls @c openSyncSession. + * 2. @ref setLocalTreeInStrand / @ref setRemoteTreeInStrand called once each by the + * scan-completion callbacks. + * 3. @ref recomputeDiffInStrand invoked on every @c recomputeSyncDiff RPC — fills the + * three @ref DiffTreeStore adjacency maps in-place, bumps @ref generation(). + * 4. @ref loadChildrenInStrand serves lazy subtree slices to the frontend tree. + * 5. Destroyed when @c closeSyncSession fires or the owning channel dies. + * + * Threading contract: every mutation and read method requires the caller to post onto + * @ref strand() first. Owned state (scan trees, diff stores, cancel flag) is never + * touched from any other thread, so no internal locks are needed. + * + * Cancellation: a captured @ref std::shared_ptr to an @ref std::atomic_bool lets + * in-flight diff walks exit early. @ref cancel() is lock-free and safe from any + * thread; it bumps the flag but does NOT post onto the strand — the running walk + * observes it at the next checkpoint. + */ +class SyncSession : public std::enable_shared_from_this +{ + public: + struct Options + { + Ids::SyncSessionId sessionId{}; + std::filesystem::path localRoot{}; + std::filesystem::path remoteRoot{}; + /// Number of scan entries (sum of both sides) above which + /// @ref SharedData::Sync::DiffSummary::heavyCompare is set to true. + std::uint64_t heavyCompareThreshold{5000}; + }; + + SyncSession(Options options, boost::asio::any_io_executor executor); + ~SyncSession(); + SyncSession(SyncSession const&) = delete; + SyncSession& operator=(SyncSession const&) = delete; + SyncSession(SyncSession&&) = delete; + SyncSession& operator=(SyncSession&&) = delete; + + Ids::SyncSessionId id() const + { + return options_.sessionId; + } + std::uint64_t generation() const + { + return generation_; + } + + boost::asio::strand& strand() + { + return strand_; + } + + /** + * @brief Thread-safe cancel flag bump. The running walk sees it at the next + * per-512-compare checkpoint. Does not post onto the strand. + */ + void cancel(); + + /** @brief Whether the cancel flag is currently set. */ + bool isCancelled() const; + + /** + * @brief Assumes ownership of a fully-built scan tree. Precondition: running on + * @ref strand(). + */ + void setLocalTreeInStrand(SharedData::Sync::ScanNode tree); + void setRemoteTreeInStrand(SharedData::Sync::ScanNode tree); + + /** @brief True once both scan trees have been handed in. */ + bool bothTreesReadyInStrand() const; + + /** + * @brief Runs the parallel merge walk, populating the three diff-tree stores, and + * returns a summary. Precondition: running on @ref strand(). + * + * @param options Diff options from the frontend settings. + * @param onProgress Invoked from the strand every 512 compares (then once more with + * the final count). Implementations typically hop to + * @c wnd_->runInJavascriptThread and emit an onSyncDiffProgress + * RPC from there. + */ + SharedData::Sync::DiffSummary recomputeDiffInStrand( + SharedData::Sync::DiffOptions const& options, + std::function onProgress + ); + + /** + * @brief Returns the direct children of @p parentRelKey in @p section (empty + * string = section root). For a one-sided directory that has not yet + * been explored, lazily walks the corresponding ScanNode subtree and + * caches the resulting DiffTreeNodes into the section store so future + * calls are O(1). Precondition: running on @ref strand(). + */ + std::vector loadChildrenInStrand( + SharedData::Sync::DiffSection section, + std::string const& parentRelKey + ); + + /** + * @brief Computes the minimal set of enqueue plan entries from a SPARSE + * selection set: an entry X in @p selectedRelKeys means "X and every + * descendant is selected". See @ref SharedData::Sync::minimizeEnqueueIndices + * for the emission rules (bulk dirs collapse; structural dirs expand + * to leaves). Precondition: running on @ref strand(). + */ + std::vector buildEnqueuePlanInStrand( + SharedData::Sync::DiffSection section, + std::unordered_set const& selectedRelKeys + ) const; + + private: + /** + * @brief Per-section result store. + * - @ref childrenByParent maps parentRelKey -> sorted DiffTreeNode list + * (the sort order is the one the merge walk used). + * - @ref emissionOrder keeps the full emission list in walk order. Used + * by @ref buildEnqueuePlanInStrand to preserve deterministic plan order + * and to run the selection-minimization over a stable sequence. + */ + struct DiffTreeStore + { + std::unordered_map nodesByRelKey{}; + std::unordered_map> childrenByParent{}; + std::vector emissionOrder{}; + SharedData::Sync::SectionSummary summary{}; + + void clear(); + }; + + Options options_; + boost::asio::any_io_executor executor_; + boost::asio::strand strand_; + std::shared_ptr> cancelled_; + + std::optional localTree_{}; + std::optional remoteTree_{}; + + DiffTreeStore uploads_{}; + DiffTreeStore downloads_{}; + DiffTreeStore deletes_{}; + + std::uint64_t generation_{0}; +}; diff --git a/backend/source/backend/CMakeLists.txt b/backend/source/backend/CMakeLists.txt index 97bb13a8..cc457d00 100644 --- a/backend/source/backend/CMakeLists.txt +++ b/backend/source/backend/CMakeLists.txt @@ -27,6 +27,7 @@ add_library( sftp/rename_operation.cpp sftp/archive_download_operation.cpp sftp/archive_upload_operation.cpp + sync/sync_session.cpp "${CMAKE_BINARY_DIR}/include/build_environment.hpp" ) diff --git a/backend/source/backend/session.cpp b/backend/source/backend/session.cpp index 7bca71ef..dfeb220c 100644 --- a/backend/source/backend/session.cpp +++ b/backend/source/backend/session.cpp @@ -67,7 +67,12 @@ void Session::start() self->registerRpcSftpAddBulkUploadOperation(); self->registerRpcSftpAddBulkDeleteOperation(); self->registerRpcSftpExistsBatch(); - self->registerRpcSftpAddSyncScanOperation(); + self->registerRpcSftpOpenSyncSession(); + self->registerRpcSftpRecomputeSyncDiff(); + self->registerRpcSftpLoadSyncDiffChildren(); + self->registerRpcSftpBuildSyncEnqueuePlan(); + self->registerRpcSftpCancelSyncDiff(); + self->registerRpcSftpCloseSyncSession(); self->registerOperationQueuePauseUnpause(); self->registerRpcSftpDeleteFiles(); self->registerRpcSftpRename(); @@ -1213,13 +1218,14 @@ void Session::registerOperationQueuePauseUnpause() } ); } -void Session::registerRpcSftpAddSyncScanOperation() +void Session::registerRpcSftpOpenSyncSession() { - on(fmt::format("Session::{}::sftp::addSyncScan", id_.value())) + on(fmt::format("Session::{}::sftp::openSyncSession", id_.value())) .perform( [weak = weak_from_this()]( RpcHelper::RpcOnce&& reply, std::string const& channelIdString, + std::string const& syncSessionIdString, std::string const& remoteScanIdString, std::string const& localScanIdString, std::string const& remotePath, @@ -1236,6 +1242,7 @@ void Session::registerRpcSftpAddSyncScanOperation() self->withSftpChannelDo( Ids::makeChannelId(channelIdString), [weak = self->weak_from_this(), + syncSessionIdString, remoteScanIdString, localScanIdString, remotePath, @@ -1250,6 +1257,7 @@ void Session::registerRpcSftpAddSyncScanOperation() self->operationQueue_->addSyncScanOperation( *channel, + Ids::makeSyncSessionId(syncSessionIdString), Ids::makeOperationId(remoteScanIdString), Ids::makeOperationId(localScanIdString), remotePath, @@ -1268,6 +1276,197 @@ void Session::registerRpcSftpAddSyncScanOperation() ); } +void Session::registerRpcSftpRecomputeSyncDiff() +{ + on(fmt::format("Session::{}::sftp::recomputeSyncDiff", id_.value())) + .perform( + [weak = weak_from_this()]( + RpcHelper::RpcOnce&& reply, + std::string const& syncSessionIdString, + SharedData::Sync::DiffOptions const& options + ) + { + auto self = weak.lock(); + if (!self || !self->operationQueue_) + return reply({{"error", "Session no longer exists"}}); + + const auto syncSessionId = Ids::makeSyncSessionId(syncSessionIdString); + auto session = self->operationQueue_->syncSession(syncSessionId); + if (!session) + return reply({{"error", "Sync session not found"}}); + + auto replyShared = std::make_shared(std::move(reply)); + boost::asio::post( + session->strand(), + [weak, session, options, syncSessionId, replyShared]() mutable + { + auto parent = weak.lock(); + const auto summary = session->recomputeDiffInStrand( + options, + [weakParent = weak, syncSessionId](std::uint64_t compared) + { + auto parent = weakParent.lock(); + if (!parent) + return; + // Hop back to the window/JS thread before emitting the RPC. + parent->wnd_->runInJavascriptThread( + [parent, syncSessionId, compared]() + { + // Matches the OperationQueue::rpcName scheme + // the frontend OperationQueue listens on. + parent->hub_->callRemote( + fmt::format( + "OperationQueue::{}::onSyncDiffProgress", + parent->id_.value() + ), + syncSessionId, + std::to_string(compared) + ); + } + ); + } + ); + + nlohmann::json payload; + SharedData::to_json(payload, summary); + (*replyShared)(std::move(payload)); + } + ); + } + ); +} + +void Session::registerRpcSftpLoadSyncDiffChildren() +{ + on(fmt::format("Session::{}::sftp::loadSyncDiffChildren", id_.value())) + .perform( + [weak = weak_from_this()]( + RpcHelper::RpcOnce&& reply, + std::string const& syncSessionIdString, + SharedData::Sync::DiffSection section, + std::string const& parentRelKey, + std::string const& expectedGenerationString + ) + { + auto self = weak.lock(); + if (!self || !self->operationQueue_) + return reply({{"error", "Session no longer exists"}}); + + const auto syncSessionId = Ids::makeSyncSessionId(syncSessionIdString); + auto session = self->operationQueue_->syncSession(syncSessionId); + if (!session) + return reply({{"error", "Sync session not found"}}); + + const std::uint64_t expectedGeneration = std::stoull(expectedGenerationString); + + auto replyShared = std::make_shared(std::move(reply)); + boost::asio::post( + session->strand(), + [session, section, parentRelKey, expectedGeneration, replyShared]() + { + if (session->generation() != expectedGeneration) + { + replyShared->error("stale generation"); + return; + } + const auto children = session->loadChildrenInStrand(section, parentRelKey); + nlohmann::json payload = nlohmann::json::array(); + for (auto const& node : children) + { + nlohmann::json entry; + SharedData::to_json(entry, node); + payload.push_back(std::move(entry)); + } + (*replyShared)(std::move(payload)); + } + ); + } + ); +} + +void Session::registerRpcSftpBuildSyncEnqueuePlan() +{ + on(fmt::format("Session::{}::sftp::buildSyncEnqueuePlan", id_.value())) + .perform( + [weak = weak_from_this()]( + RpcHelper::RpcOnce&& reply, + std::string const& syncSessionIdString, + SharedData::Sync::DiffSection section, + std::vector const& selectedRelKeys, + std::string const& expectedGenerationString + ) + { + auto self = weak.lock(); + if (!self || !self->operationQueue_) + return reply({{"error", "Session no longer exists"}}); + + const auto syncSessionId = Ids::makeSyncSessionId(syncSessionIdString); + auto session = self->operationQueue_->syncSession(syncSessionId); + if (!session) + return reply({{"error", "Sync session not found"}}); + + const std::uint64_t expectedGeneration = std::stoull(expectedGenerationString); + auto selected = std::unordered_set{selectedRelKeys.begin(), selectedRelKeys.end()}; + + auto replyShared = std::make_shared(std::move(reply)); + boost::asio::post( + session->strand(), + [session, section, selected = std::move(selected), expectedGeneration, replyShared]() + { + if (session->generation() != expectedGeneration) + { + replyShared->error("stale generation"); + return; + } + const auto plan = session->buildEnqueuePlanInStrand(section, selected); + nlohmann::json payload = nlohmann::json::array(); + for (auto const& entry : plan) + { + nlohmann::json j; + SharedData::to_json(j, entry); + payload.push_back(std::move(j)); + } + (*replyShared)(std::move(payload)); + } + ); + } + ); +} + +void Session::registerRpcSftpCancelSyncDiff() +{ + on(fmt::format("Session::{}::sftp::cancelSyncDiff", id_.value())) + .perform( + [weak = weak_from_this()](RpcHelper::RpcOnce&& reply, std::string const& syncSessionIdString) + { + auto self = weak.lock(); + if (!self || !self->operationQueue_) + return reply({{"error", "Session no longer exists"}}); + + const auto syncSessionId = Ids::makeSyncSessionId(syncSessionIdString); + if (auto session = self->operationQueue_->syncSession(syncSessionId); session) + session->cancel(); + reply({{"success", true}}); + } + ); +} + +void Session::registerRpcSftpCloseSyncSession() +{ + on(fmt::format("Session::{}::sftp::closeSyncSession", id_.value())) + .perform( + [weak = weak_from_this()](RpcHelper::RpcOnce&& reply, std::string const& syncSessionIdString) + { + auto self = weak.lock(); + if (!self || !self->operationQueue_) + return reply({{"error", "Session no longer exists"}}); + + self->operationQueue_->closeSyncSession(Ids::makeSyncSessionId(syncSessionIdString)); + reply({{"success", true}}); + } + ); +} + void Session::registerRpcSftpPreDeleteChecks() { on(fmt::format("Session::{}::sftp::preDeleteChecks", id_.value())) diff --git a/backend/source/backend/sftp/local_scan_operation.cpp b/backend/source/backend/sftp/local_scan_operation.cpp index 8d599978..dfc1a58a 100644 --- a/backend/source/backend/sftp/local_scan_operation.cpp +++ b/backend/source/backend/sftp/local_scan_operation.cpp @@ -31,6 +31,7 @@ LocalScanOperation::LocalScanOperation(ScanOperationOptions options) , respectIgnoreFiles_{options.respectIgnoreFiles} , recursive_{options.recursive} , ignoreHidden_{options.ignoreHidden} + , buildTree_{options.buildTree} {} LocalScanOperation::~LocalScanOperation() = default; @@ -198,27 +199,29 @@ std::expected LocalSc } case (Running): { - return withWalkerDo( + auto stepWalker = [this](auto& walker) -> std::expected + { + if (walker.completed()) { - if (walker.completed()) - { - Log::info("LocalScanOperation: Scan of '{}' completed.", localPath_.generic_string()); - state_ = Completed; - return WorkStatus::Complete; - } + Log::info("LocalScanOperation: Scan of '{}' completed.", localPath_.generic_string()); + state_ = Completed; + return WorkStatus::Complete; + } - auto result = walker.walk(); - if (!result.has_value()) - { - Log::error("LocalScanOperation: Failed to scan directory: {}", result.error().toString()); - return enterErrorState(result.error()); - } - // -1, because the walker includes the base/root dir of the search: - progressCallback_(walker.totalBytes(), walker.currentIndex(), walker.totalEntries() - 1); - return WorkStatus::MoreWork; + auto result = walker.walk(); + if (!result.has_value()) + { + Log::error("LocalScanOperation: Failed to scan directory: {}", result.error().toString()); + return enterErrorState(result.error()); } - ); + // -1, because the walker includes the base/root dir of the search: + progressCallback_(walker.totalBytes(), walker.currentIndex(), walker.totalEntries() - 1); + return WorkStatus::MoreWork; + }; + if (buildTree_) + return withTreeWalkerDo(stepWalker); + return withWalkerDo(stepWalker); } case (Prepared): case (Preparing): diff --git a/backend/source/backend/sftp/operation_queue.cpp b/backend/source/backend/sftp/operation_queue.cpp index 6551950a..cc859b31 100644 --- a/backend/source/backend/sftp/operation_queue.cpp +++ b/backend/source/backend/sftp/operation_queue.cpp @@ -3,13 +3,14 @@ #include #include #include -#include #include #include #include #include #include +#include + #include #include @@ -643,14 +644,10 @@ bool OperationQueue::workQueue(std::deque(operation.get()); - auto entries = scan->ejectEntries(); - // Pre-compute absolute fullPaths from parent chain - for (auto& entry : entries) - entry.fullPath = SharedData::fullPath(entries, entry); - const auto totalBytes = scan->totalBytes(); + auto tree = scan->ejectScanTree(); auto callback = std::move(cbIt->second); syncScanCallbacks_.erase(cbIt); - callback(std::move(entries), totalBytes); + callback(std::move(tree)); } else Log::error("Scan operation completed but no following BulkOperation to set results to."); @@ -688,13 +685,10 @@ bool OperationQueue::workQueue(std::deque(operation.get()); - auto entries = scan->ejectEntries(); - for (auto& entry : entries) - entry.fullPath = SharedData::fullPath(entries, entry); - const auto totalBytes = scan->totalBytes(); + auto tree = scan->ejectScanTree(); auto callback = std::move(cbIt->second); syncScanCallbacks_.erase(cbIt); - callback(std::move(entries), totalBytes); + callback(std::move(tree)); } else Log::error("LocalScan operation completed but no following BulkOperation to set results to."); @@ -1866,6 +1860,7 @@ std::expected OperationQueue::addRenameOperation( void OperationQueue::addSyncScanOperation( SecureShell::SftpSession& sftp, + Ids::SyncSessionId syncSessionId, Ids::OperationId remoteScanId, Ids::OperationId localScanId, std::filesystem::path const& remotePath, @@ -1875,42 +1870,57 @@ void OperationQueue::addSyncScanOperation( bool ignoreHidden ) { - // Register completion callbacks that emit onSyncScanResult to the frontend. + // Create (or replace) the SyncSession. A fresh id from the frontend is the + // norm; re-opening over an existing id is handled as "discard previous state". + auto session = std::make_shared( + SyncSession::Options{ + .sessionId = syncSessionId, + .localRoot = localPath, + .remoteRoot = remotePath, + }, + executor_ + ); + syncSessions_[syncSessionId.value()] = session; + + // Scan completion callbacks hand the ScanNode trees into the session on its + // own strand, then emit onSyncScanPhaseDone to the frontend. syncScanCallbacks_[remoteScanId.value()] = - [weak = weak_from_this(), remoteScanId](std::vector entries, std::uint64_t totalBytes) + [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}] + (SharedData::Sync::ScanNode tree) { auto self = weak.lock(); - if (!self) + auto sess = weakSession.lock(); + if (!self || !sess) return; - self->hub_->callRemote( - self->rpcName("onSyncScanResult"), - SharedData::SyncScanResult{ - .operationId = remoteScanId, - .isLocal = false, - .totalBytes = totalBytes, - .entries = std::move(entries), + boost::asio::post( + sess->strand(), + [sess, tree = std::move(tree)]() mutable + { + sess->setRemoteTreeInStrand(std::move(tree)); } ); + self->hub_->callRemote(self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/false); }; syncScanCallbacks_[localScanId.value()] = - [weak = weak_from_this(), localScanId](std::vector entries, std::uint64_t totalBytes) + [weak = weak_from_this(), syncSessionId, weakSession = std::weak_ptr{session}] + (SharedData::Sync::ScanNode tree) { auto self = weak.lock(); - if (!self) + auto sess = weakSession.lock(); + if (!self || !sess) return; - self->hub_->callRemote( - self->rpcName("onSyncScanResult"), - SharedData::SyncScanResult{ - .operationId = localScanId, - .isLocal = true, - .totalBytes = totalBytes, - .entries = std::move(entries), + boost::asio::post( + sess->strand(), + [sess, tree = std::move(tree)]() mutable + { + sess->setLocalTreeInStrand(std::move(tree)); } ); + self->hub_->callRemote(self->rpcName("onSyncScanPhaseDone"), syncSessionId, /*isLocal=*/true); }; - // Queue remote scan + // Queue remote scan — build-tree mode. auto remoteScan = std::make_unique( sftp, ScanOperation::ScanOperationOptions{ @@ -1920,6 +1930,7 @@ void OperationQueue::addSyncScanOperation( .respectIgnoreFiles = respectIgnoreFiles, .recursive = recursive, .ignoreHidden = ignoreHidden, + .buildTree = true, } ); priorityOperations_.emplace_back(remoteScanId, std::move(remoteScan)); @@ -1934,13 +1945,14 @@ void OperationQueue::addSyncScanOperation( } ); - // Queue local scan + // Queue local scan — build-tree mode. auto localScan = std::make_unique(LocalScanOperation::ScanOperationOptions{ .progressCallback = makeScanProgressCallback("onLocalScanProgress", localScanId), .localPath = localPath, .respectIgnoreFiles = respectIgnoreFiles, .recursive = recursive, .ignoreHidden = ignoreHidden, + .buildTree = true, }); priorityOperations_.emplace_back(localScanId, std::move(localScan)); @@ -1955,6 +1967,22 @@ void OperationQueue::addSyncScanOperation( ); } +std::shared_ptr OperationQueue::syncSession(Ids::SyncSessionId const& sessionId) const +{ + const auto iter = syncSessions_.find(sessionId.value()); + return iter == syncSessions_.end() ? nullptr : iter->second; +} + +void OperationQueue::closeSyncSession(Ids::SyncSessionId const& sessionId) +{ + const auto iter = syncSessions_.find(sessionId.value()); + if (iter == syncSessions_.end()) + return; + if (iter->second) + iter->second->cancel(); + syncSessions_.erase(iter); +} + void OperationQueue::registerRpc() { on(rpcName("isPaused")) diff --git a/backend/source/backend/sftp/scan_operation.cpp b/backend/source/backend/sftp/scan_operation.cpp index 0c9230bf..b5cc675d 100644 --- a/backend/source/backend/sftp/scan_operation.cpp +++ b/backend/source/backend/sftp/scan_operation.cpp @@ -15,6 +15,7 @@ ScanOperation::ScanOperation(SecureShell::SftpSession& sftp, ScanOperationOption , respectIgnoreFiles_{options.respectIgnoreFiles} , recursive_{options.recursive} , ignoreHidden_{options.ignoreHidden} + , buildTree_{options.buildTree} {} ScanOperation::~ScanOperation() = default; @@ -142,27 +143,31 @@ std::expected ScanOperation::wo } case (Running): { - return withWalkerDo( + // Shared step logic — generic over both walker shapes since + // BaseDirectoryWalker and the concrete walkers expose the same surface. + auto stepWalker = [this](auto& walker) -> std::expected + { + if (walker.completed()) { - if (walker.completed()) - { - Log::info("ScanOperation: Scan of '{}' completed.", remotePath_.generic_string()); - state_ = Completed; - return WorkStatus::Complete; - } - - auto result = walker.walk(); - if (!result.has_value()) - { - Log::error("ScanOperation: Failed to scan directory: {}", result.error().toString()); - return enterErrorState(result.error()); - } - // -1, because the walker includes the base/root dir of the search: - progressCallback_(walker.totalBytes(), walker.currentIndex(), walker.totalEntries() - 1); - return WorkStatus::MoreWork; + Log::info("ScanOperation: Scan of '{}' completed.", remotePath_.generic_string()); + state_ = Completed; + return WorkStatus::Complete; } - ); + + auto result = walker.walk(); + if (!result.has_value()) + { + Log::error("ScanOperation: Failed to scan directory: {}", result.error().toString()); + return enterErrorState(result.error()); + } + // -1, because the walker includes the base/root dir of the search: + progressCallback_(walker.totalBytes(), walker.currentIndex(), walker.totalEntries() - 1); + return WorkStatus::MoreWork; + }; + if (buildTree_) + return withTreeWalkerDo(stepWalker); + return withWalkerDo(stepWalker); } case (Prepared): case (Preparing): diff --git a/backend/source/backend/sync/sync_session.cpp b/backend/source/backend/sync/sync_session.cpp new file mode 100644 index 00000000..9c52c026 --- /dev/null +++ b/backend/source/backend/sync/sync_session.cpp @@ -0,0 +1,522 @@ +#include + +#include + +#include +#include +#include +#include +#include + +using namespace SharedData::Sync; + +namespace +{ + /** + * @brief DiffSink implementation that fans each emission into the matching section + * store. Also feeds the running compared-count up to an external lambda and + * consults the session-owned cancel flag. + * + * All methods run on the session strand (inherited from the caller of + * @ref diffScanTrees); no locking needed. + */ + class SessionDiffSink final : public DiffSink + { + public: + using ProgressFn = std::function; + using StoreRef = std::unordered_map>&; + using NodeMapRef = std::unordered_map&; + using OrderRef = std::vector&; + + struct Side + { + NodeMapRef nodesByRelKey; + StoreRef childrenByParent; + OrderRef emissionOrder; + SectionSummary& summary; + }; + + SessionDiffSink( + Side uploads, + Side downloads, + Side deletes, + std::shared_ptr> cancelled, + ProgressFn onProgress + ) + : uploads_{uploads} + , downloads_{downloads} + , deletes_{deletes} + , cancelled_{std::move(cancelled)} + , onProgress_{std::move(onProgress)} + {} + + void emitUpload(DiffTreeNode node, std::string const& parentRelKey) override + { + emitInto(uploads_, std::move(node), parentRelKey); + } + void emitDownload(DiffTreeNode node, std::string const& parentRelKey) override + { + emitInto(downloads_, std::move(node), parentRelKey); + } + void emitDelete(DiffTreeNode node, std::string const& parentRelKey) override + { + emitInto(deletes_, std::move(node), parentRelKey); + } + + void onProgress(std::uint64_t entriesCompared) override + { + lastCompared_ = entriesCompared; + if (onProgress_) + onProgress_(entriesCompared); + } + + bool cancelled() const override + { + return cancelled_ && cancelled_->load(std::memory_order_acquire); + } + + std::uint64_t lastCompared() const + { + return lastCompared_; + } + + private: + void emitInto(Side& side, DiffTreeNode node, std::string const& parentRelKey) + { + side.summary.itemCount += 1; + if (parentRelKey.empty()) + side.summary.rootChildCount += 1; + + // A leaf contributes its own transferable side; a one-sided directory + // contributes the pre-computed subtree total (already on the correct side). + if (node.isDirectory) + side.summary.transferBytes += node.descendantByteTotal; + else + side.summary.transferBytes += node.hasLocalSide ? node.localSize : node.remoteSize; + + const auto leafKey = node.relKey; + side.emissionOrder.push_back(leafKey); + side.nodesByRelKey[leafKey] = node; + side.childrenByParent[parentRelKey].push_back(std::move(node)); + } + + Side uploads_; + Side downloads_; + Side deletes_; + std::shared_ptr> cancelled_; + ProgressFn onProgress_; + std::uint64_t lastCompared_{0}; + }; +} + +namespace +{ + /** @brief Returns the parent relKey of @p relKey (everything before the last + * '/'), or an empty string when @p relKey has no separator. + */ + std::string parentOf(std::string const& relKey) + { + const auto slash = relKey.rfind('/'); + if (slash == std::string::npos) + return {}; + return relKey.substr(0, slash); + } + + std::string basenameOf(std::string const& relKey) + { + const auto slash = relKey.rfind('/'); + if (slash == std::string::npos) + return relKey; + return relKey.substr(slash + 1); + } + + /** @brief Resolves @p relKey against @p root by walking the sorted children + * at each level. Returns nullptr if any segment can't be matched. + */ + SharedData::Sync::ScanNode const* findScanNode( + SharedData::Sync::ScanNode const& root, + std::string const& relKey + ) + { + if (relKey.empty()) + return &root; + auto const* cur = &root; + std::size_t start = 0; + while (start < relKey.size()) + { + const auto slash = relKey.find('/', start); + const auto seg = (slash == std::string::npos) + ? std::string_view{relKey}.substr(start) + : std::string_view{relKey}.substr(start, slash - start); + auto iter = std::lower_bound( + cur->children.begin(), + cur->children.end(), + seg, + [](SharedData::Sync::ScanNode const& n, std::string_view s) { return n.name < s; } + ); + if (iter == cur->children.end() || iter->name != seg) + return nullptr; + cur = &*iter; + if (slash == std::string::npos) + break; + start = slash + 1; + } + return cur; + } + + /** @brief Builds a one-sided child DiffTreeNode from a ScanNode, carrying the + * parent's @p action through. + */ + SharedData::Sync::DiffTreeNode makeOneSidedChildRow( + SharedData::Sync::ScanNode const& scan, + std::string const& childRelKey, + SharedData::Sync::Action action, + bool isLocalSide + ) + { + const bool isDir = scan.type == SharedData::FileType::Directory; + return SharedData::Sync::DiffTreeNode{ + .relKey = childRelKey, + .name = scan.name, + .action = action, + .isDirectory = isDir, + .hasLocalSide = isLocalSide, + .hasRemoteSide = !isLocalSide, + .localSize = isLocalSide ? scan.size : 0, + .remoteSize = isLocalSide ? 0 : scan.size, + .localMtime = isLocalSide ? scan.mtime : 0, + .remoteMtime = isLocalSide ? 0 : scan.mtime, + .directChildCount = isDir ? static_cast(scan.children.size()) : 0u, + .descendantItemCount = isDir ? scan.subtreeFileCount : 1ull, + .descendantByteTotal = isDir ? scan.subtreeByteTotal : scan.size, + .isStructural = false, + }; + } +} + +void SyncSession::DiffTreeStore::clear() +{ + nodesByRelKey.clear(); + childrenByParent.clear(); + emissionOrder.clear(); + summary = SectionSummary{}; +} + +SyncSession::SyncSession(Options options, boost::asio::any_io_executor executor) + : options_{std::move(options)} + , executor_{executor} + , strand_{boost::asio::make_strand(executor_)} + , cancelled_{std::make_shared>(false)} +{} + +SyncSession::~SyncSession() = default; + +void SyncSession::cancel() +{ + if (cancelled_) + cancelled_->store(true, std::memory_order_release); +} + +bool SyncSession::isCancelled() const +{ + return cancelled_ && cancelled_->load(std::memory_order_acquire); +} + +void SyncSession::setLocalTreeInStrand(ScanNode tree) +{ + localTree_ = std::move(tree); +} + +void SyncSession::setRemoteTreeInStrand(ScanNode tree) +{ + remoteTree_ = std::move(tree); +} + +bool SyncSession::bothTreesReadyInStrand() const +{ + return localTree_.has_value() && remoteTree_.has_value(); +} + +DiffSummary SyncSession::recomputeDiffInStrand( + DiffOptions const& options, + std::function onProgress +) +{ + uploads_.clear(); + downloads_.clear(); + deletes_.clear(); + ++generation_; + + DiffSummary summary{}; + summary.sessionId = options_.sessionId; + summary.generation = generation_; + summary.cancelled = false; + + if (!localTree_ || !remoteTree_) + { + // Nothing to diff yet; return zeroed summary so callers don't block. + return summary; + } + + const std::uint64_t totalCount = localTree_->subtreeFileCount + remoteTree_->subtreeFileCount + + localTree_->children.size() + remoteTree_->children.size(); + summary.heavyCompare = totalCount > options_.heavyCompareThreshold; + + // Reset the cancel flag at the start of each walk so a previous cancel doesn't + // bleed into the new run. A fresh cancel during the walk still works because it + // stores `true` into the same shared atomic. + cancelled_->store(false, std::memory_order_release); + + SessionDiffSink sink{ + SessionDiffSink::Side{ + uploads_.nodesByRelKey, uploads_.childrenByParent, + uploads_.emissionOrder, uploads_.summary}, + SessionDiffSink::Side{ + downloads_.nodesByRelKey, downloads_.childrenByParent, + downloads_.emissionOrder, downloads_.summary}, + SessionDiffSink::Side{ + deletes_.nodesByRelKey, deletes_.childrenByParent, + deletes_.emissionOrder, deletes_.summary}, + cancelled_, + std::move(onProgress) + }; + + diffScanTrees(*localTree_, *remoteTree_, options, sink); + + // Synthesize structural ancestor rows for every deep emission so the + // frontend can build the tree hierarchy above differing leaves. A diff on + // 'a/b/c.txt' emits only 'c.txt' at parent 'a/b' — without this pass, the + // frontend can't represent 'a' or 'a/b'. + auto synthesizeStructuralAncestors = [](DiffTreeStore& store) { + // Snapshot the walk-order emissions — we'll be appending to emissionOrder. + const auto initialEmissions = store.emissionOrder; + std::unordered_set alreadyHave(initialEmissions.begin(), initialEmissions.end()); + + std::vector newAncestors; + for (auto const& relKey : initialEmissions) + { + auto ancestor = parentOf(relKey); + while (!ancestor.empty() && !alreadyHave.contains(ancestor)) + { + alreadyHave.insert(ancestor); + newAncestors.push_back(ancestor); + ancestor = parentOf(ancestor); + } + } + // Sort shortest-first so when we build children lists we always have + // the parent record in place. + std::sort(newAncestors.begin(), newAncestors.end(), [](auto const& a, auto const& b) { + return a.size() < b.size(); + }); + + for (auto const& relKey : newAncestors) + { + DiffTreeNode structural{ + .relKey = relKey, + .name = basenameOf(relKey), + .action = Action::Upload, + .isDirectory = true, + .hasLocalSide = true, + .hasRemoteSide = true, + .localSize = 0, + .remoteSize = 0, + .localMtime = 0, + .remoteMtime = 0, + .directChildCount = 0, + .descendantItemCount = 0, + .descendantByteTotal = 0, + .isStructural = true, + }; + store.nodesByRelKey[relKey] = structural; + store.childrenByParent[parentOf(relKey)].push_back(structural); + store.emissionOrder.push_back(relKey); + } + + // Re-sort each affected childrenByParent list by name so synthesized + // ancestors slot into the right position alongside emitted leaves. + for (auto& [parent, children] : store.childrenByParent) + { + std::sort(children.begin(), children.end(), [](DiffTreeNode const& a, DiffTreeNode const& b) { + return a.name < b.name; + }); + } + + // Fill in directChildCount for synthesized rows now that every + // children list is finalized. + for (auto const& relKey : newAncestors) + { + if (auto iter = store.childrenByParent.find(relKey); iter != store.childrenByParent.end()) + { + store.nodesByRelKey[relKey].directChildCount = + static_cast(iter->second.size()); + } + // Propagate the updated count into the parent's child list too. + const auto parent = parentOf(relKey); + auto parentListIt = store.childrenByParent.find(parent); + if (parentListIt == store.childrenByParent.end()) + continue; + for (auto& child : parentListIt->second) + { + if (child.relKey == relKey) + child.directChildCount = store.nodesByRelKey[relKey].directChildCount; + } + } + }; + synthesizeStructuralAncestors(uploads_); + synthesizeStructuralAncestors(downloads_); + synthesizeStructuralAncestors(deletes_); + + summary.uploads = uploads_.summary; + summary.downloads = downloads_.summary; + summary.deletes = deletes_.summary; + summary.entriesCompared = sink.lastCompared(); + summary.cancelled = sink.cancelled(); + return summary; +} + +std::vector SyncSession::loadChildrenInStrand( + DiffSection section, + std::string const& parentRelKey +) +{ + auto& store = + section == DiffSection::Upload ? uploads_ : + section == DiffSection::Download ? downloads_ : + deletes_; + + if (const auto iter = store.childrenByParent.find(parentRelKey); + iter != store.childrenByParent.end()) + { + return iter->second; + } + + // No cached children — candidate for lazy expansion of a one-sided + // directory. Check that parentRelKey names a known one-sided dir node. + const auto nodeIt = store.nodesByRelKey.find(parentRelKey); + if (nodeIt == store.nodesByRelKey.end()) + return {}; + auto const& parentNode = nodeIt->second; + if (!parentNode.isDirectory || parentNode.isStructural) + return {}; + // Exactly-one-sided is the only case where the diff walk bailed early; + // two-sided differing directories don't make sense as a single action. + const bool oneSided = parentNode.hasLocalSide ^ parentNode.hasRemoteSide; + if (!oneSided) + return {}; + + // Pick the side that has actual ScanNode data and find the subtree. + auto const& rootOpt = parentNode.hasLocalSide ? localTree_ : remoteTree_; + if (!rootOpt) + return {}; + auto const* scanNode = findScanNode(*rootOpt, parentRelKey); + if (!scanNode || scanNode->type != SharedData::FileType::Directory) + return {}; + + // Emit each child as a one-sided DiffTreeNode with the parent's action. + std::vector produced; + produced.reserve(scanNode->children.size()); + for (auto const& scanChild : scanNode->children) + { + const auto childRel = parentRelKey.empty() + ? scanChild.name + : parentRelKey + "/" + scanChild.name; + auto row = + makeOneSidedChildRow(scanChild, childRel, parentNode.action, parentNode.hasLocalSide); + store.nodesByRelKey[childRel] = row; + store.emissionOrder.push_back(childRel); + produced.push_back(std::move(row)); + } + store.childrenByParent[parentRelKey] = produced; + return produced; +} + +namespace +{ + bool nodeIsBulkDir(SharedData::Sync::DiffTreeNode const& node) + { + if (!node.isDirectory) + return false; + switch (node.action) + { + case SharedData::Sync::Action::Upload: + return !node.hasRemoteSide; + case SharedData::Sync::Action::Download: + return !node.hasLocalSide; + case SharedData::Sync::Action::DeleteLocal: + case SharedData::Sync::Action::DeleteRemote: + return true; + } + return false; + } + + /** + * @brief Resolves the local + remote absolute paths that apply to @p node. + * For actions that only touch one side, the other path is still set + * to the "would-be" destination — keeps the frontend enqueue wiring + * uniform. + */ + std::pair resolveAbsPaths( + SharedData::Sync::DiffTreeNode const& node, + std::filesystem::path const& localRoot, + std::filesystem::path const& remoteRoot + ) + { + const auto rel = std::filesystem::path{node.relKey}; + return {(localRoot / rel).generic_string(), (remoteRoot / rel).generic_string()}; + } +} + +std::vector SyncSession::buildEnqueuePlanInStrand( + SharedData::Sync::DiffSection section, + std::unordered_set const& selectedRelKeys +) const +{ + auto const& store = + section == DiffSection::Upload ? uploads_ : + section == DiffSection::Download ? downloads_ : + deletes_; + + std::vector views; + views.reserve(store.emissionOrder.size()); + for (auto const& relKey : store.emissionOrder) + { + const auto iter = store.nodesByRelKey.find(relKey); + if (iter == store.nodesByRelKey.end()) + continue; + views.push_back(SharedData::Sync::MinimizerItemView{ + .relKey = relKey, + .isBulkDir = nodeIsBulkDir(iter->second), + }); + } + + const auto keptIndices = SharedData::Sync::minimizeEnqueueIndices(views, selectedRelKeys); + + std::vector plan; + plan.reserve(keptIndices.size()); + for (auto idx : keptIndices) + { + auto const& relKey = views[idx].relKey; + const auto nodeIt = store.nodesByRelKey.find(relKey); + if (nodeIt == store.nodesByRelKey.end()) + continue; + auto const& node = nodeIt->second; + auto [localAbs, remoteAbs] = resolveAbsPaths(node, options_.localRoot, options_.remoteRoot); + + const std::uint64_t sizeBytes = node.isDirectory + ? 0ull + : (node.hasLocalSide ? node.localSize : node.remoteSize); + + plan.push_back(SharedData::Sync::EnqueuePlanEntry{ + .relKey = relKey, + .action = node.action, + .localAbsPath = std::move(localAbs), + .remoteAbsPath = std::move(remoteAbs), + .sizeBytes = sizeBytes, + .mtime = node.hasLocalSide ? node.localMtime : node.remoteMtime, + .mtimeNsec = 0, + .isDirectory = node.isDirectory, + }); + } + + return plan; +} diff --git a/dependencies/5cript-nui-components b/dependencies/5cript-nui-components index d5bc172d..3ec2d750 160000 --- a/dependencies/5cript-nui-components +++ b/dependencies/5cript-nui-components @@ -1 +1 @@ -Subproject commit d5bc172d26c24d3fcd2fe87f67b156c13687d8c7 +Subproject commit 3ec2d750b8f4d3f05a425deba0f39831da810d1f diff --git a/frontend/include/frontend/session_components/operation_queue.hpp b/frontend/include/frontend/session_components/operation_queue.hpp index 1038fbff..7599765f 100644 --- a/frontend/include/frontend/session_components/operation_queue.hpp +++ b/frontend/include/frontend/session_components/operation_queue.hpp @@ -10,7 +10,6 @@ #include #include #include -#include #include #include #include @@ -228,7 +227,19 @@ class OperationQueue /** @brief Hides the minimized-sync restore button (e.g. when the dialog is closed or reopened). */ void hideMinimizedSync(); - void enqueueSyncScans( + /** + * @brief Opens a backend SyncSession and kicks both scans. The two scans build + * sorted ScanNode trees on the backend; no entry payloads cross RPC. + * + * The frontend learns when each side finishes via the per-side phase-done + * callback (separate from scan-progress events). A backend + * @c onSyncDiffProgress stream — registered here too — is routed to the + * provider-level progress callback so the Comparing phase can animate. + * + * @param syncSessionId Pre-allocated session id; reused across later RPCs. + */ + void openSyncSession( + Ids::SyncSessionId syncSessionId, std::filesystem::path localPath, std::filesystem::path remotePath, bool respectIgnoreFiles, @@ -236,10 +247,47 @@ class OperationQueue bool ignoreHidden, std::function onRemoteProgress, std::function onLocalProgress, - std::function onRemoteComplete, - std::function onLocalComplete + std::function onScanPhaseDone, + std::function onDiffProgress ); + /** + * @brief Clears per-session routing state installed by @ref openSyncSession. + * Called when the provider tears down (dialog close or session drop). + */ + void clearSyncSessionRouting(Ids::SyncSessionId syncSessionId); + + // Thin pass-throughs to @ref FileEngine for the sync-session RPC surface. + // Keep them on @ref OperationQueue so @ref BackendSyncProvider doesn't need + // direct FileEngine access — matches the rest of the file-op RPCs here. + + void recomputeSyncDiff( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffOptions options, + std::function onSummary + ); + + void loadSyncDiffChildren( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::string const& parentRelKey, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected + ); + + void buildSyncEnqueuePlan( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::vector selectedRelKeys, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected + ); + + void cancelSyncDiff(Ids::SyncSessionId syncSessionId); + void closeSyncSession(Ids::SyncSessionId syncSessionId); + /** * @brief Extracts a resumable descriptor for every currently-in-flight * (not yet completed) card in both the priority and normal queues. diff --git a/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp new file mode 100644 index 00000000..d854109d --- /dev/null +++ b/frontend/include/frontend/sync_dialog/backend_sync_provider.hpp @@ -0,0 +1,114 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include + +class OperationQueue; + +/** + * @brief Frontend handle to a backend-resident SyncSession. + * + * Owns the @ref Ids::SyncSessionId and wraps the RPC call surface: + * + * - @ref open queues both scans, bridges @c onSyncScanPhaseDone to @p onBothListed. + * - @ref recompute triggers a backend diff walk and asks for the resulting summary. + * The stored @ref generation_ is advanced; stale @ref loadChildren responses + * issued against a previous generation are silently dropped. + * - @ref loadChildren feeds the frontend's lazy @ref Tree. + * - @ref cancelDiff flips the backend cancel flag. + * - @ref close releases backend memory. Idempotent. + * + * The provider owns the lifecycle — callers should construct it once per sync flow + * and destroy it when the dialog closes. The destructor calls @ref close. + */ +class BackendSyncProvider +{ + public: + explicit BackendSyncProvider(OperationQueue* queue); + ~BackendSyncProvider(); + BackendSyncProvider(BackendSyncProvider const&) = delete; + BackendSyncProvider& operator=(BackendSyncProvider const&) = delete; + BackendSyncProvider(BackendSyncProvider&&) = delete; + BackendSyncProvider& operator=(BackendSyncProvider&&) = delete; + + Ids::SyncSessionId sessionId() const + { + return sessionId_; + } + std::uint64_t generation() const + { + return generation_; + } + + /** + * @brief Kicks off both scans. @p onBothListed fires after both sides have + * reported phase-done (the order between them isn't significant). + */ + void open( + std::filesystem::path localPath, + std::filesystem::path remotePath, + bool respectIgnoreFiles, + bool recursive, + bool ignoreHidden, + std::function onLocalListing, + std::function onRemoteListing, + std::function onBothListed, + std::function onDiffProgress + ); + + /** + * @brief Runs a backend diff with @p options and hands back a @ref DiffSummary. + * The summary carries the new generation id; cache it for subsequent + * @ref loadChildren calls. + */ + void recompute( + SharedData::Sync::DiffOptions options, + std::function onSummary + ); + + /** + * @brief Lazy loader for the @ref ScriptNuiComponents::Tree. Rejections include + * stale-generation responses; callers should silently drop those. + */ + void loadChildren( + SharedData::Sync::DiffSection section, + std::string const& parentRelKey, + std::function)> onResolved, + std::function onRejected + ); + + /** + * @brief Asks the backend to collapse @p selectedRelKeys into a minimal + * enqueue plan for @p section. Uses the cached @ref generation_ so + * stale requests (after a recompute) are rejected by the server. + */ + void buildEnqueuePlan( + SharedData::Sync::DiffSection section, + std::vector selectedRelKeys, + std::function)> onResolved, + std::function onRejected + ); + + /** @brief Flips the backend cancel flag; an in-flight diff walk bails at next checkpoint. */ + void cancelDiff(); + + /** @brief Releases backend-side memory. Idempotent. */ + void close(); + + private: + OperationQueue* queue_; + Ids::SyncSessionId sessionId_; + std::uint64_t generation_{0}; + bool closed_{false}; +}; diff --git a/frontend/include/frontend/sync_dialog/sync_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_dialog.hpp index cb724b4b..538f4d20 100644 --- a/frontend/include/frontend/sync_dialog/sync_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_dialog.hpp @@ -1,6 +1,8 @@ #pragma once -#include +#include +#include +#include #include #include @@ -8,7 +10,6 @@ #include #include #include -#include class ConfirmDialog; class OperationQueue; @@ -23,41 +24,45 @@ class SyncDialog SyncDialog(SyncDialog&&); SyncDialog& operator=(SyncDialog&&); - /** @brief Opens the dialog with pre-scanned directory listings and computes the diff. + /** + * @brief Opens the dialog against a backend-resident sync session whose first + * diff has already completed. * - * @param localPath Absolute path of the local directory root. - * @param remotePath Absolute path of the remote directory root. - * @param localEntries Flat entry list from a LocalScanOperation (fullPaths pre-computed). - * @param remoteEntries Flat entry list from a ScanOperation (fullPaths pre-computed). + * @param provider Non-owning handle to the session; must outlive the dialog. + * @param summary The @ref DiffSummary returned from the initial recompute. + * Used to seed footer totals and drive root-tree layout. + * @param localPath Local scan root (for path column formatting only — data is + * served lazily from @p provider). + * @param remotePath Remote scan root. */ void open( + BackendSyncProvider* provider, + SharedData::Sync::DiffSummary summary, std::filesystem::path localPath, - std::filesystem::path remotePath, - std::vector localEntries, - std::vector remoteEntries + std::filesystem::path remotePath ); - /** @brief Sets the callback invoked when the user clicks Recompare. - * The callback receives local/remote paths and an onResult function that should be - * called once the new comparison is complete, passing the resulting entry lists. - * - * @param callback Callable with signature - * (std::filesystem::path local, std::filesystem::path remote, - * std::function onResult). + /** + * @brief Scan + diff settings snapshot handed to the Recompare callback. + * The scan flags (respectIgnoreFiles/recursive/ignoreHidden) govern + * the backend listing; the diff options are the first recompute's + * inputs. */ - void setOnRecompare( - std::function, - std::vector - )> - )> callback - ); + struct RecompareRequest + { + bool respectIgnoreFiles{true}; + bool recursive{true}; + bool ignoreHidden{false}; + SharedData::Sync::DiffOptions diffOptions{}; + }; + + /** + * @brief Called when the user clicks the Recompare button. Session wires + * this to close the existing provider, re-run the scan-and-diff flow + * with the given settings, then @ref open the dialog again against a + * fresh session. + */ + void setOnRecompareRequested(std::function callback); Nui::ElementRenderer operator()(); diff --git a/frontend/include/frontend/sync_dialog/sync_item.hpp b/frontend/include/frontend/sync_dialog/sync_item.hpp deleted file mode 100644 index 3862239a..00000000 --- a/frontend/include/frontend/sync_dialog/sync_item.hpp +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include -#include - -#include -#include -#include - -enum class SyncItemAction -{ - Upload, - Download, - DeleteLocal, - DeleteRemote -}; - -struct SyncItem -{ - SyncItemAction action; - std::optional localItem; - std::optional remoteItem; - // null = not yet enqueued; 0.0–1.0 = in progress; > 1.0 = completed - std::shared_ptr> progress; - /// Relative path from the sync root, with `/` separators. Doubles as a stable - /// identifier when building the tree view and when matching items after a - /// recompare rebuilds the list. - std::string relKey{}; -}; diff --git a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp index 322d3c37..add693d6 100644 --- a/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp +++ b/frontend/include/frontend/sync_dialog/sync_progress_dialog.hpp @@ -1,6 +1,8 @@ #pragma once -#include +#include +#include +#include #include #include @@ -8,7 +10,6 @@ #include #include #include -#include class OperationQueue; @@ -22,26 +23,36 @@ class SyncProgressDialog SyncProgressDialog(SyncProgressDialog&&); SyncProgressDialog& operator=(SyncProgressDialog&&); - /** @brief Opens the progress dialog and begins scanning both sides. + /** + * @brief Opens the progress dialog, runs both scans via @p provider, then + * immediately triggers a backend diff. @p onDone fires after both the + * scans and the first diff have landed, passing the @ref DiffSummary + * the frontend should seed its tree from. * - * @param localPath Root of the local directory being compared. - * @param remotePath Root of the remote directory being compared. - * @param onDone Called with (localEntries, remoteEntries) when both scans finish. - * Not called if the dialog is cancelled. + * The dialog keeps showing the Comparing spinner during the diff walk; + * light diffs finish within a frame so the phase effectively flashes. + * + * @param provider Non-owning handle to the sync session. Lifetime must + * outlive this call (typically owned by @c Session). + * @param initialOptions The diff options to apply for the first recompute. + * @param onDone Called exactly once on success. Not invoked on cancel. */ void open( + BackendSyncProvider* provider, std::filesystem::path localPath, std::filesystem::path remotePath, bool respectIgnoreFiles, bool recursive, bool ignoreHidden, - std::function localEntries, - std::vector remoteEntries - )> onDone + SharedData::Sync::DiffOptions initialOptions, + std::function onDone ); - /** @brief Cancels an in-progress scan and closes the dialog. */ + /** + * @brief Cancels the in-progress scan or diff and closes the dialog. Safe to + * call before @ref open or after @ref open completes (no-op in those + * cases). + */ void cancel(); Nui::ElementRenderer operator()(); diff --git a/frontend/include/frontend/terminal/file_engine.hpp b/frontend/include/frontend/terminal/file_engine.hpp index 6670fdfc..307ad7b0 100644 --- a/frontend/include/frontend/terminal/file_engine.hpp +++ b/frontend/include/frontend/terminal/file_engine.hpp @@ -3,6 +3,10 @@ #include #include #include +#include +#include +#include +#include #include #include #include @@ -32,19 +36,18 @@ class FileEngine createDirectory(std::filesystem::path const& path, std::function onComplete); void createFile(std::filesystem::path const& path, std::function onComplete); - /** @brief Queues both a remote and a local scan as priority operations for sync comparison. - * The pre-generated IDs are used by the backend; the frontend registers listeners - * for those IDs before calling this so that no progress events are missed. + /** + * @brief Opens a backend-resident SyncSession and queues both scans. The scans + * build sorted ScanNode trees; the session diff-then-serves lazy subtrees. * - * @param localPath Local directory root. - * @param remotePath Remote directory root. - * @param remoteScanId Pre-generated operation ID for the remote scan. - * @param localScanId Pre-generated operation ID for the local scan. - * @param onComplete Called on success/failure with (success, info). + * @param syncSessionId Pre-generated session id used for all follow-up RPCs. + * @param remoteScanId Operation id for the remote scan card. + * @param localScanId Operation id for the local scan card. */ - void addSyncScans( + void openSyncSession( std::filesystem::path const& localPath, std::filesystem::path const& remotePath, + Ids::SyncSessionId syncSessionId, Ids::OperationId remoteScanId, Ids::OperationId localScanId, bool respectIgnoreFiles, @@ -53,6 +56,52 @@ class FileEngine std::function onComplete ); + /** + * @brief Runs (or re-runs) the diff on an open sync session and returns the + * summary. @p onSummary receives a failed/empty summary with + * `cancelled=true` if the call fails. + */ + void recomputeSyncDiff( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffOptions options, + std::function onSummary + ); + + /** + * @brief Loads direct children of @p parentRelKey in @p section. @p generation + * must match the most recent @ref recomputeSyncDiff reply or the server + * rejects as stale — the caller then discards. + */ + void loadSyncDiffChildren( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::string const& parentRelKey, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected + ); + + /** + * @brief Asks the backend to collapse the selected relKey SPARSE set in @p section + * into a minimal enqueue plan. An entry X in @p selectedRelKeys means "X + * and every descendant selected"; the backend-resident minimizer handles + * bulk-dir / structural-dir rules. + */ + void buildSyncEnqueuePlan( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::vector selectedRelKeys, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected + ); + + /** @brief Flips the backend cancel flag. The running walk exits at next checkpoint. */ + void cancelSyncDiff(Ids::SyncSessionId syncSessionId); + + /** @brief Releases backend-side memory for @p syncSessionId. */ + void closeSyncSession(Ids::SyncSessionId syncSessionId); + void addDownload( NuiFileExplorer::Item const& remotePath, NuiFileExplorer::Item const& localPath, diff --git a/frontend/source/frontend/CMakeLists.txt b/frontend/source/frontend/CMakeLists.txt index c39a08a8..91aad185 100644 --- a/frontend/source/frontend/CMakeLists.txt +++ b/frontend/source/frontend/CMakeLists.txt @@ -56,6 +56,7 @@ target_sources( session_components/session_snapshot_manager.cpp sync_dialog/sync_dialog.cpp sync_dialog/sync_progress_dialog.cpp + sync_dialog/backend_sync_provider.cpp terminal/terminal_channel.cpp terminal/frontend_session_manager.cpp terminal/executing_engine.cpp diff --git a/frontend/source/frontend/session.cpp b/frontend/source/frontend/session.cpp index e898dd86..5f619448 100644 --- a/frontend/source/frontend/session.cpp +++ b/frontend/source/frontend/session.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include #include @@ -73,6 +74,14 @@ struct Session::Implementation std::unique_ptr layoutInitializer; SyncDialog syncDialog; SyncProgressDialog syncProgressDialog; + // Holds the active BackendSyncProvider between progress-dialog open and + // sync-dialog close. Destroyed (→ provider.close() → backend session + // released) when replaced or explicitly reset. + std::unique_ptr syncProvider_; + // Last local+remote paths given to startSyncFlow; used by the dialog's + // Recompare path to re-issue the flow without asking the user again. + std::filesystem::path lastSyncLocalPath_{}; + std::filesystem::path lastSyncRemotePath_{}; std::unique_ptr connectionLossOverlay; std::unique_ptr snapshotManager; @@ -126,28 +135,72 @@ struct Session::Implementation , onReconnectCancel{std::move(params.onReconnectCancel)} , onReconnectNow{std::move(params.onReconnectNow)} { - syncDialog.setOnRecompare( - [this]( - std::filesystem::path loc, - std::filesystem::path rem, - bool respectIgnoreFiles, - bool recursive, - bool ignoreHidden, - std::function, - std::vector - )> onResult - ) + // Recompare: rerun the whole scan-and-diff flow with the dialog's current + // settings, reusing the cached local+remote roots. The old provider is + // replaced by a fresh one; the backend session it held is closed on + // destruction. + syncDialog.setOnRecompareRequested( + [this](SyncDialog::RecompareRequest req) { - syncProgressDialog.open( - std::move(loc), std::move(rem), respectIgnoreFiles, recursive, ignoreHidden, std::move(onResult) + if (lastSyncLocalPath_.empty() || lastSyncRemotePath_.empty()) + return; + startSyncFlowImpl( + lastSyncLocalPath_, + lastSyncRemotePath_, + req.respectIgnoreFiles, + req.recursive, + req.ignoreHidden, + req.diffOptions ); - Nui::globalEventContext.executeActiveEventsImmediately(); } ); fileExplorerPanel.dropLayoutMetadata(sessionLayoutId); } + + /** + * @brief Runs one scan-and-diff flow: replaces any existing provider, drives + * the progress dialog through Listing → Comparing, then opens the + * sync dialog with the resulting summary. + * + * Callers: + * - file-explorer "synchronize" click (initial open), and + * - sync-dialog "recompare" click (uses the cached paths). + */ + void startSyncFlowImpl( + std::filesystem::path localPath, + std::filesystem::path remotePath, + bool respectIgnoreFiles, + bool recursive, + bool ignoreHidden, + SharedData::Sync::DiffOptions initialDiffOptions + ) + { + lastSyncLocalPath_ = localPath; + lastSyncRemotePath_ = remotePath; + + // Replace the old provider before opening the progress dialog so its + // backend session (if any) is closed before the new one opens. + syncProvider_ = std::make_unique(&operationQueue); + + syncProgressDialog.open( + syncProvider_.get(), + localPath, + remotePath, + respectIgnoreFiles, + recursive, + ignoreHidden, + initialDiffOptions, + [this, localPath, remotePath](SharedData::Sync::DiffSummary summary) + { + syncDialog.open( + syncProvider_.get(), std::move(summary), localPath, remotePath + ); + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ); + Nui::globalEventContext.executeActiveEventsImmediately(); + } }; int Session::tabId() const @@ -415,23 +468,20 @@ void Session::setupFileGrid() impl_->fileExplorerPanel.setup(); // Sync dialog + progress dialog stay on Session; wire them through the - // panel's synchronize callback. + // panel's synchronize callback. Initial flow uses default scan + diff + // settings; the sync dialog's own settings drive subsequent recomputes and + // recompares. impl_->fileExplorerPanel.setOnSynchronize( [this](std::filesystem::path loc, std::filesystem::path rem) { - impl_->syncProgressDialog.open( - loc, - rem, - true, - true, - false, - [this, loc, rem](auto localEntries, auto remoteEntries) - { - impl_->syncDialog.open(loc, rem, std::move(localEntries), std::move(remoteEntries)); - Nui::globalEventContext.executeActiveEventsImmediately(); - } + impl_->startSyncFlowImpl( + std::move(loc), + std::move(rem), + /*respectIgnoreFiles=*/true, + /*recursive=*/true, + /*ignoreHidden=*/false, + SharedData::Sync::DiffOptions{} ); - Nui::globalEventContext.executeActiveEventsImmediately(); } ); } diff --git a/frontend/source/frontend/session_components/operation_queue.cpp b/frontend/source/frontend/session_components/operation_queue.cpp index 482f3758..71f52123 100644 --- a/frontend/source/frontend/session_components/operation_queue.cpp +++ b/frontend/source/frontend/session_components/operation_queue.cpp @@ -91,9 +91,18 @@ struct OperationQueue::Implementation // bulk up/download, or the aggregate id for bulk delete). Auto-erased on // completion, same as transferProgressCallbacks. std::unordered_map> bulkProgressCallbacks; - // Keyed by operationId.value(); used for sync scan progress + completion routing. + // Keyed by operationId.value(); used for per-operation sync scan progress events. std::unordered_map> syncScanProgressCallbacks; - std::unordered_map> syncScanCompletionCallbacks; + + // Keyed by syncSessionId.value(); routes the backend's phase-done / diff-progress + // streams to the provider. Cleared via @ref clearSyncSessionRouting. + struct SyncSessionRouting + { + std::array scanIds{}; // [0]=remote, [1]=local + std::function onScanPhaseDone; + std::function onDiffProgress; + }; + std::unordered_map syncSessionRouting; Nui::ListenRemover pausedListener{}; // Minimized-sync restore button state. `minimizedSyncVisible` drives @@ -602,28 +611,41 @@ void OperationQueue::activate(std::shared_ptr fileEngine, Ids::Sessi impl_->onUpdate.push_back( Nui::RpcClient::autoRegisterFunction( - fmt::format("OperationQueue::{}::onSyncScanResult", impl_->sessionId.value()), - [this](Nui::val val) + fmt::format("OperationQueue::{}::onSyncScanPhaseDone", impl_->sessionId.value()), + [this](std::string const& sessionIdString, bool isLocal) { - SharedData::SyncScanResult result{}; + const auto routingIt = impl_->syncSessionRouting.find(sessionIdString); + if (routingIt == impl_->syncSessionRouting.end()) + return; + // The per-operation progress entry for this side is no longer interesting. + const auto& scanId = routingIt->second.scanIds[isLocal ? 1 : 0]; + impl_->syncScanProgressCallbacks.erase(scanId.value()); + if (routingIt->second.onScanPhaseDone) + routingIt->second.onScanPhaseDone(isLocal); + } + ) + ); + + impl_->onUpdate.push_back( + Nui::RpcClient::autoRegisterFunction( + fmt::format("OperationQueue::{}::onSyncDiffProgress", impl_->sessionId.value()), + [this](std::string const& sessionIdString, std::string const& comparedString) + { + const auto routingIt = impl_->syncSessionRouting.find(sessionIdString); + if (routingIt == impl_->syncSessionRouting.end()) + return; + if (!routingIt->second.onDiffProgress) + return; + std::uint64_t compared = 0; try { - const auto json = nlohmann::json::parse(Nui::JSON::stringify(val)); - result = json.get(); + compared = std::stoull(comparedString); } - catch (std::exception const& exc) + catch (std::exception const&) { - Log::error("Failed to parse SyncScanResult: {}", exc.what()); return; } - const auto cbIt = impl_->syncScanCompletionCallbacks.find(result.operationId.value()); - if (cbIt != impl_->syncScanCompletionCallbacks.end()) - { - auto callback = std::move(cbIt->second); - impl_->syncScanCompletionCallbacks.erase(cbIt); - impl_->syncScanProgressCallbacks.erase(result.operationId.value()); - callback(std::move(result)); - } + routingIt->second.onDiffProgress(compared); } ) ); @@ -1363,7 +1385,8 @@ void OperationQueue::unpause() ); } -void OperationQueue::enqueueSyncScans( +void OperationQueue::openSyncSession( + Ids::SyncSessionId syncSessionId, std::filesystem::path localPath, std::filesystem::path remotePath, bool respectIgnoreFiles, @@ -1371,28 +1394,33 @@ void OperationQueue::enqueueSyncScans( bool ignoreHidden, std::function onRemoteProgress, std::function onLocalProgress, - std::function onRemoteComplete, - std::function onLocalComplete + std::function onScanPhaseDone, + std::function onDiffProgress ) { if (!impl_->fileEngine) { - Log::error("No file engine set for operation queue, cannot enqueue sync scans"); + Log::error("No file engine set for operation queue, cannot open sync session"); return; } const auto remoteScanId = Ids::generateOperationId(); const auto localScanId = Ids::generateOperationId(); - // Register callbacks before hitting the backend so no events are missed. + // Register per-operation scan-progress + per-session routing before hitting the + // backend so no events are missed. impl_->syncScanProgressCallbacks[remoteScanId.value()] = std::move(onRemoteProgress); impl_->syncScanProgressCallbacks[localScanId.value()] = std::move(onLocalProgress); - impl_->syncScanCompletionCallbacks[remoteScanId.value()] = std::move(onRemoteComplete); - impl_->syncScanCompletionCallbacks[localScanId.value()] = std::move(onLocalComplete); + impl_->syncSessionRouting[syncSessionId.value()] = Implementation::SyncSessionRouting{ + .scanIds = {remoteScanId, localScanId}, + .onScanPhaseDone = std::move(onScanPhaseDone), + .onDiffProgress = std::move(onDiffProgress), + }; - impl_->fileEngine->addSyncScans( + impl_->fileEngine->openSyncSession( localPath, remotePath, + syncSessionId, remoteScanId, localScanId, respectIgnoreFiles, @@ -1402,7 +1430,7 @@ void OperationQueue::enqueueSyncScans( { if (!success) Log::error( - "Failed to enqueue sync scans (remoteId={}, localId={}): {}", + "Failed to open sync session (remoteId={}, localId={}): {}", remoteScanId.value(), localScanId.value(), info @@ -1411,6 +1439,79 @@ void OperationQueue::enqueueSyncScans( ); } +void OperationQueue::clearSyncSessionRouting(Ids::SyncSessionId syncSessionId) +{ + const auto routingIt = impl_->syncSessionRouting.find(syncSessionId.value()); + if (routingIt == impl_->syncSessionRouting.end()) + return; + // Drop the per-scan-operation progress entries too in case a phase-done never fired. + for (auto const& scanId : routingIt->second.scanIds) + impl_->syncScanProgressCallbacks.erase(scanId.value()); + impl_->syncSessionRouting.erase(routingIt); +} + +void OperationQueue::recomputeSyncDiff( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffOptions options, + std::function onSummary +) +{ + if (!impl_->fileEngine) + return; + impl_->fileEngine->recomputeSyncDiff(syncSessionId, std::move(options), std::move(onSummary)); +} + +void OperationQueue::loadSyncDiffChildren( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::string const& parentRelKey, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected +) +{ + if (!impl_->fileEngine) + return; + impl_->fileEngine->loadSyncDiffChildren( + syncSessionId, section, parentRelKey, generation, std::move(onResolved), std::move(onRejected) + ); +} + +void OperationQueue::buildSyncEnqueuePlan( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::vector selectedRelKeys, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected +) +{ + if (!impl_->fileEngine) + return; + impl_->fileEngine->buildSyncEnqueuePlan( + syncSessionId, + section, + std::move(selectedRelKeys), + generation, + std::move(onResolved), + std::move(onRejected) + ); +} + +void OperationQueue::cancelSyncDiff(Ids::SyncSessionId syncSessionId) +{ + if (!impl_->fileEngine) + return; + impl_->fileEngine->cancelSyncDiff(syncSessionId); +} + +void OperationQueue::closeSyncSession(Ids::SyncSessionId syncSessionId) +{ + if (!impl_->fileEngine) + return; + impl_->fileEngine->closeSyncSession(syncSessionId); +} + void OperationQueue::createRemoteDirectory( std::filesystem::path const& path, std::function onComplete diff --git a/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp new file mode 100644 index 00000000..9887f424 --- /dev/null +++ b/frontend/source/frontend/sync_dialog/backend_sync_provider.cpp @@ -0,0 +1,163 @@ +#include +#include + +#include + +#include + +BackendSyncProvider::BackendSyncProvider(OperationQueue* queue) + : queue_{queue} + , sessionId_{Ids::generateSyncSessionId()} +{} + +BackendSyncProvider::~BackendSyncProvider() +{ + close(); +} + +void BackendSyncProvider::open( + std::filesystem::path localPath, + std::filesystem::path remotePath, + bool respectIgnoreFiles, + bool recursive, + bool ignoreHidden, + std::function onLocalListing, + std::function onRemoteListing, + std::function onBothListed, + std::function onDiffProgress +) +{ + if (!queue_) + { + Log::error("BackendSyncProvider::open called with no OperationQueue"); + return; + } + + // Shared state for the two-phase-done gate. Both sides must finish before we + // unblock the caller's onBothListed. + struct PhaseGate + { + bool local{false}; + bool remote{false}; + std::function onBothListed; + }; + auto gate = std::make_shared(); + gate->onBothListed = std::move(onBothListed); + + queue_->openSyncSession( + sessionId_, + std::move(localPath), + std::move(remotePath), + respectIgnoreFiles, + recursive, + ignoreHidden, + std::move(onRemoteListing), + std::move(onLocalListing), + [gate](bool isLocal) + { + if (isLocal) + gate->local = true; + else + gate->remote = true; + if (gate->local && gate->remote && gate->onBothListed) + { + auto cb = std::move(gate->onBothListed); + gate->onBothListed = nullptr; + cb(); + } + }, + std::move(onDiffProgress) + ); +} + +void BackendSyncProvider::recompute( + SharedData::Sync::DiffOptions options, + std::function onSummary +) +{ + if (!queue_ || closed_) + return; + + queue_->recomputeSyncDiff( + sessionId_, + std::move(options), + [this, onSummary = std::move(onSummary)](SharedData::Sync::DiffSummary summary) mutable + { + generation_ = summary.generation; + onSummary(std::move(summary)); + } + ); +} + +void BackendSyncProvider::loadChildren( + SharedData::Sync::DiffSection section, + std::string const& parentRelKey, + std::function)> onResolved, + std::function onRejected +) +{ + if (!queue_ || closed_) + return; + + const auto expectedGeneration = generation_; + queue_->loadSyncDiffChildren( + sessionId_, + section, + parentRelKey, + expectedGeneration, + // Drop stale responses silently: if the generation moved on before the RPC + // returned the frontend already has a fresher view, and surfacing a stale + // result would confuse the tree's keyed merge. + [this, expectedGeneration, onResolved = std::move(onResolved)]( + std::vector nodes) mutable + { + if (expectedGeneration != generation_) + return; + onResolved(std::move(nodes)); + }, + std::move(onRejected) + ); +} + +void BackendSyncProvider::buildEnqueuePlan( + SharedData::Sync::DiffSection section, + std::vector selectedRelKeys, + std::function)> onResolved, + std::function onRejected +) +{ + if (!queue_ || closed_) + return; + + const auto expectedGeneration = generation_; + queue_->buildSyncEnqueuePlan( + sessionId_, + section, + std::move(selectedRelKeys), + expectedGeneration, + [this, expectedGeneration, onResolved = std::move(onResolved)]( + std::vector plan) mutable + { + if (expectedGeneration != generation_) + return; + onResolved(std::move(plan)); + }, + std::move(onRejected) + ); +} + +void BackendSyncProvider::cancelDiff() +{ + if (!queue_ || closed_) + return; + queue_->cancelSyncDiff(sessionId_); +} + +void BackendSyncProvider::close() +{ + if (closed_ || !queue_) + return; + closed_ = true; + queue_->clearSyncSessionRouting(sessionId_); + queue_->closeSyncSession(sessionId_); +} diff --git a/frontend/source/frontend/sync_dialog/sync_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_dialog.cpp index d22a45dc..a0f43b57 100644 --- a/frontend/source/frontend/sync_dialog/sync_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_dialog.cpp @@ -1,5 +1,5 @@ #include -#include +#include #include #include #include @@ -20,7 +20,6 @@ #include #include #include -#include #include #include @@ -36,21 +35,22 @@ #include -#include #include -#include #include -#include +#include +#include +#include #include #include #include #include +#include #include -#include -#include +#include #include +#include #include #include @@ -59,6 +59,8 @@ using namespace std::string_literals; namespace { using SyncDirection = SharedData::Sync::Direction; + using SharedData::Sync::DiffSection; + using SharedData::Sync::DiffTreeNode; std::string formatMtime(std::uint64_t mtime) { @@ -67,52 +69,12 @@ namespace return fmt::format("{:%Y-%m-%d}", floor(tp)); } - struct SyncTotals - { - std::size_t count{0}; - std::uint64_t bytes{0}; - }; - - /** @brief Returns the byte count of the side that's actually being transferred - * for @p itm. Directories (size==0) and missing sides contribute 0. - */ - std::uint64_t transferBytes(SyncItem const& itm) - { - switch (itm.action) - { - case SyncItemAction::Upload: - return itm.localItem ? itm.localItem->size : 0ull; - case SyncItemAction::Download: - return itm.remoteItem ? itm.remoteItem->size : 0ull; - case SyncItemAction::DeleteLocal: - return itm.localItem ? itm.localItem->size : 0ull; - case SyncItemAction::DeleteRemote: - return itm.remoteItem ? itm.remoteItem->size : 0ull; - } - return 0ull; - } - - /** @brief Sums item count + transfer bytes for @p items. When @p selected - * is non-null, only items whose relKey is in the set contribute (the - * set holds leaf relKeys only — directory rows naturally get skipped). + /** @brief Renders one (local or remote) cell of a diff row from the relevant side + * of a @ref DiffTreeNode. @p hasSide gates visibility; when false the + * cell renders as the spacer that keeps the grid columns aligned. */ - SyncTotals computeTotals( - std::vector const& items, - std::unordered_set const* selected = nullptr - ) - { - SyncTotals total{}; - for (auto const& itm : items) - { - if (selected && !selected->contains(itm.relKey)) - continue; - ++total.count; - total.bytes += transferBytes(itm); - } - return total; - } - - Nui::ElementRenderer renderItemCell(std::optional const& item, bool alignRight) + Nui::ElementRenderer + renderCellFromNode(DiffTreeNode const& node, bool isRemote, bool alignRight) { using namespace Nui::Elements; using namespace Nui::Attributes; @@ -120,33 +82,29 @@ namespace using Nui::Elements::div; using Nui::Elements::span; - if (!item) + const bool hasSide = isRemote ? node.hasRemoteSide : node.hasLocalSide; + if (!hasSide) { return div{class_ = fmt::format("sync-diff-cell empty {}", alignRight ? "sync-diff-cell-right" : "")}(); } - const auto& entry = *item; - const auto sizeStr = entry.type == SharedData::FileType::Directory - ? std::string{} - : Utility::formatBytes(static_cast(entry.size)); - const auto mtimeStr = entry.mtime > 0 ? formatMtime(entry.mtime) : std::string{}; - // The tree already communicates hierarchy via indent + chevron; showing - // the full relative path here just duplicates that and wastes row space. - const auto name = entry.path.filename().generic_string(); + const std::uint64_t size = isRemote ? node.remoteSize : node.localSize; + const std::uint64_t mtime = isRemote ? node.remoteMtime : node.localMtime; + const std::string sizeStr = + node.isDirectory ? std::string{} : Utility::formatBytes(static_cast(size)); + const std::string mtimeStr = mtime > 0 ? formatMtime(mtime) : std::string{}; return div{ class_ = fmt::format("sync-diff-cell {}", alignRight ? "sync-diff-cell-right" : ""), - "title"_attr = entry.fullPath.string() + "title"_attr = node.relKey }( - span{class_ = "name"}(name), - [&]() -> Nui::ElementRenderer - { + span{class_ = "name"}(node.name), + [&]() -> Nui::ElementRenderer { if (sizeStr.empty()) return Nui::nil(); return span{class_ = "meta"}(sizeStr); }(), - [&]() -> Nui::ElementRenderer - { + [&]() -> Nui::ElementRenderer { if (mtimeStr.empty()) return Nui::nil(); return span{class_ = "meta date"}(mtimeStr); @@ -154,6 +112,98 @@ namespace ); } + DiffTreeNode const* userDataAsDiffTreeNode(std::any const& userData) + { + return std::any_cast(&userData); + } + + /** @brief Parent relKey (everything before the last '/'), or empty when @p + * relKey has no separator. Used by the sparse-selection helpers. + */ + std::string sparseParentOf(std::string const& relKey) + { + const auto slash = relKey.rfind('/'); + return slash == std::string::npos ? std::string{} : relKey.substr(0, slash); + } + + /** @brief Walks every parent prefix of @p relKey and returns true if any of + * them is present in @p set. "Effective selection" in the sparse + * model: an entry X in the set implies every descendant of X. + */ + bool sparseAnyAncestorInSet( + std::string const& relKey, + std::unordered_set const& set + ) + { + std::string_view view{relKey}; + while (true) + { + const auto slash = view.rfind('/'); + if (slash == std::string_view::npos) + return false; + view.remove_suffix(view.size() - slash); + if (set.contains(std::string{view})) + return true; + } + } + + /** @brief Same walk, but returns the covering ancestor's relKey (or empty + * when none is found). Used for fill-out on uncheck. + */ + std::string sparseFindCoveringAncestor( + std::string const& relKey, + std::unordered_set const& set + ) + { + std::string_view view{relKey}; + while (true) + { + const auto slash = view.rfind('/'); + if (slash == std::string_view::npos) + return {}; + view.remove_suffix(view.size() - slash); + std::string candidate{view}; + if (set.contains(candidate)) + return candidate; + } + } + + /** @brief True when @p set contains any entry strictly below @p relKey. + * Linear in |set|. Used only to flip an Unchecked directory row + * into Indeterminate when a descendant was selected individually. + */ + bool sparseAnyDescendantInSet( + std::string const& relKey, + std::unordered_set const& set + ) + { + const std::string prefix = relKey + "/"; + for (auto const& key : set) + { + if (key.size() > prefix.size() && key.starts_with(prefix)) + return true; + } + return false; + } + + /** @brief True when the row should be anchored on the right column — used by + * mid-tree "directory only" rows whose side-placement must match the + * leaves underneath. Upload = left, Download/DeleteRemote = right, + * DeleteLocal = left. + */ + bool actionContentOnRight(SharedData::Sync::Action action) + { + switch (action) + { + case SharedData::Sync::Action::Upload: + case SharedData::Sync::Action::DeleteLocal: + return false; + case SharedData::Sync::Action::Download: + case SharedData::Sync::Action::DeleteRemote: + return true; + } + return false; + } } // ---- Implementation --------------------------------------------------------- @@ -161,15 +211,11 @@ namespace struct SyncDialog::Implementation { Nui::Observed open_{false}; - // When true, the dialog DOM stays mounted with full state intact but is - // hidden; a restore button in the OperationQueue header brings it back. Nui::Observed minimized_{false}; std::filesystem::path localPath_{}; std::filesystem::path remotePath_{}; - // Cached scan results (set by open()) - std::vector localEntries_{}; - std::vector remoteEntries_{}; + BackendSyncProvider* provider_{nullptr}; // Settings Nui::Observed directionStr_{"Both"s}; @@ -181,13 +227,28 @@ struct SyncDialog::Implementation Nui::Observed actionDownload_{true}; Nui::Observed actionDelete_{false}; - // Diff item lists - Nui::Observed> uploadItems_{}; - Nui::Observed> downloadItems_{}; - Nui::Observed> deleteItems_{}; - - // Per-tree selection sets (leaf NodeIds only; directory tristate is - // computed by the tree from these). Shared with the tree Options. + // Summary — drives footer totals and section counts. Backend-authoritative; + // the frontend never needs the full entry list to compute these. + Nui::Observed summary_{}; + + // Per-section progress observers keyed by relKey. Allocated lazily when a row + // is enqueued (arrow click or bulk synchronize); the row renderer reads the + // relKey's observer back to paint the gradient. When a bulk directory is + // enqueued, every descendant relKey gets the *same* observer so rows inside + // the subtree share the parent's progress. + using ProgressObserver = std::shared_ptr>; + std::unordered_map uploadProgress_{}; + std::unordered_map downloadProgress_{}; + std::unordered_map deleteProgress_{}; + + // Drives the row renderer's observe() so the gradient repaints when any + // progress observer updates. Bumped after every provider response. Cheap: + // one extra int per paint. + Nui::Observed progressEpoch_{0}; + + // Selection sets (leaf relKeys only — directories derive their tristate from + // descendants). Shared with the Tree via Options::selected so the tree + // toolbar buttons manipulate the same storage. std::shared_ptr>> uploadSelected_{ std::make_shared>>()}; std::shared_ptr>> downloadSelected_{ @@ -195,8 +256,6 @@ struct SyncDialog::Implementation std::shared_ptr>> deleteSelected_{ std::make_shared>>()}; - // One tree per diff list; holds per-node expansion state across recompares. - // Options (row renderer etc.) are built in the ctor and never reassigned. ScriptNuiComponents::Tree uploadTree_{}; ScriptNuiComponents::Tree downloadTree_{}; ScriptNuiComponents::Tree deleteTree_{}; @@ -207,18 +266,7 @@ struct SyncDialog::Implementation ConfirmDialog* confirmDialog_; OperationQueue* operationQueue_; - std::function, - std::vector - )> - )> - onRecompare_{}; + std::function onRecompareRequested_{}; explicit Implementation(ConfirmDialog* confirmDialog, OperationQueue* operationQueue) : confirmDialog_{confirmDialog} @@ -228,635 +276,752 @@ struct SyncDialog::Implementation initTrees(); } + /** @brief Direction bit — "right" means the download/remote-leaning cell. */ + static DiffSection const& sectionForTree(bool isUpload, bool isDownload) + { + static const DiffSection upload = DiffSection::Upload; + static const DiffSection download = DiffSection::Download; + static const DiffSection del = DiffSection::Delete; + if (isUpload) return upload; + if (isDownload) return download; + return del; + } + + std::unordered_map& progressMapFor(DiffSection section) + { + switch (section) + { + case DiffSection::Upload: + return uploadProgress_; + case DiffSection::Download: + return downloadProgress_; + case DiffSection::Delete: + return deleteProgress_; + } + return uploadProgress_; + } + + std::shared_ptr>>& selectionFor(DiffSection section) + { + switch (section) + { + case DiffSection::Upload: + return uploadSelected_; + case DiffSection::Download: + return downloadSelected_; + case DiffSection::Delete: + return deleteSelected_; + } + return uploadSelected_; + } + + ScriptNuiComponents::Tree& treeFor(DiffSection section) + { + switch (section) + { + case DiffSection::Upload: + return uploadTree_; + case DiffSection::Download: + return downloadTree_; + case DiffSection::Delete: + return deleteTree_; + } + return uploadTree_; + } + void initTrees() { namespace Snc = ScriptNuiComponents; uploadTree_ = Snc::Tree{Snc::Tree::Options{ - .rowContent = makeTreeRowRenderer(uploadItems_, /*mirrored=*/false), - .rowAttributes = makeTreeRowAttributes(), + .rowContent = makeTreeRowRenderer(DiffSection::Upload), + .childrenLoader = makeChildrenLoader(DiffSection::Upload), .showCheckboxes = true, .showIcons = false, .selected = uploadSelected_, + .selectionStateResolver = makeStateResolver(DiffSection::Upload), + .toggleSelection = makeToggleSelection(DiffSection::Upload), + .selectAllAction = makeSelectAllAction(DiffSection::Upload), + .deselectAllAction = makeDeselectAllAction(), .showCollapseAllButton = true, .showSelectAllButton = true, .showDeselectAllButton = true, }}; downloadTree_ = Snc::Tree{Snc::Tree::Options{ - .rowContent = makeTreeRowRenderer(downloadItems_, /*mirrored=*/true), - .rowAttributes = makeTreeRowAttributes(), + .rowContent = makeTreeRowRenderer(DiffSection::Download), + .childrenLoader = makeChildrenLoader(DiffSection::Download), .showCheckboxes = true, .showIcons = false, .mirror = true, .selected = downloadSelected_, + .selectionStateResolver = makeStateResolver(DiffSection::Download), + .toggleSelection = makeToggleSelection(DiffSection::Download), + .selectAllAction = makeSelectAllAction(DiffSection::Download), + .deselectAllAction = makeDeselectAllAction(), .showCollapseAllButton = true, .showSelectAllButton = true, .showDeselectAllButton = true, }}; deleteTree_ = Snc::Tree{Snc::Tree::Options{ - .rowContent = makeTreeRowRenderer(deleteItems_, /*mirrored=*/false), - .rowAttributes = makeTreeRowAttributes(), + .rowContent = makeTreeRowRenderer(DiffSection::Delete), + .childrenLoader = makeChildrenLoader(DiffSection::Delete), .showCheckboxes = true, .showIcons = false, .selected = deleteSelected_, + .selectionStateResolver = makeStateResolver(DiffSection::Delete), + .toggleSelection = makeToggleSelection(DiffSection::Delete), + .selectAllAction = makeSelectAllAction(DiffSection::Delete), + .deselectAllAction = makeDeselectAllAction(), .showCollapseAllButton = true, .showSelectAllButton = true, .showDeselectAllButton = true, }}; } - ScriptNuiComponents::Tree::RowContentRenderer - makeTreeRowRenderer(Nui::Observed>& listObs, bool mirrored); - - ScriptNuiComponents::Tree::RowAttributeProvider makeTreeRowAttributes(); - - /** @brief Feeds the current item vectors into the trees. Called after any - * modification of uploadItems_/downloadItems_/deleteItems_. The - * trees' keyed merge preserves per-node expansion state. + /** @brief Builds the tri-state resolver for sparse semantics: self-in-set + * OR ancestor-in-set ⇒ Checked; self not covered but descendant in + * set ⇒ Indeterminate; otherwise Unchecked. Cheap per call — one + * ancestor walk up to the root plus at most one linear scan of the + * (sparse) set. */ - void refreshTrees(); - - void enqueueSingleByRelKey(Nui::Observed>& list, std::string const& relKey); + std::function + makeStateResolver(DiffSection section) + { + return [this, section](ScriptNuiComponents::Tree::NodeId const& id) { + using State = ScriptNuiComponents::Tree::SelectionState; + auto const& set = selectionFor(section)->value(); + if (set.contains(id)) + return State::Checked; + if (sparseAnyAncestorInSet(id, set)) + return State::Checked; + if (sparseAnyDescendantInSet(id, set)) + return State::Indeterminate; + return State::Unchecked; + }; + } - void recomputeDiff() + /** @brief Builds the sparse toggle callback. See the design note in + * @c toggleSparseSelection for the algorithm. + */ + std::function&)> + makeToggleSelection(DiffSection section) { - const SharedData::Sync::DiffOptions diffOptions{ - .direction = direction_, - .actionUpload = actionUpload_.value(), - .actionDownload = actionDownload_.value(), - .actionDelete = actionDelete_.value(), - .recursive = recursive_.value(), - .ignoreHidden = ignoreHidden_.value(), + return [this, section]( + ScriptNuiComponents::Tree::NodeId const& id, + bool nowSelected, + std::unordered_set& set) { + toggleSparseSelection(section, id, nowSelected, set); }; - auto diff = SharedData::Sync::computeSyncDiff( - localPath_, remotePath_, localEntries_, remoteEntries_, diffOptions - ); + } - const auto actionToSyncItemAction = [](SharedData::Sync::Action action) { - switch (action) - { - case SharedData::Sync::Action::Upload: - return SyncItemAction::Upload; - case SharedData::Sync::Action::Download: - return SyncItemAction::Download; - case SharedData::Sync::Action::DeleteLocal: - return SyncItemAction::DeleteLocal; - case SharedData::Sync::Action::DeleteRemote: - return SyncItemAction::DeleteRemote; - } - return SyncItemAction::Upload; + std::function&)> + makeSelectAllAction(DiffSection section) + { + return [this, section](std::unordered_set& set) { + set.clear(); + auto const roots = treeFor(section).childrenOf(std::string{}); + for (auto const& rootId : roots) + set.insert(rootId); }; + } - const auto toFileExplorerItem = [](SharedData::DirectoryEntry entry, std::string const& relKey) { - entry.path = std::filesystem::path{relKey}; - return NuiFileExplorer::Item{std::move(entry)}; + std::function&)> makeDeselectAllAction() + { + return [](std::unordered_set& set) { + set.clear(); }; + } - const auto convert = [&](std::vector&& diffEntries) { - std::vector result; - result.reserve(diffEntries.size()); - for (auto& diffEntry : diffEntries) + /** @brief The sparse-toggle core. On check: drop any descendants already in + * the set (they become implied by @p id), insert @p id unless an + * ancestor already covers it, then try to collapse up — replace a + * full set of siblings at the parent level with the parent itself. + * + * On uncheck: if @p id is in the set, erase it and stop; else walk + * up to the covering ancestor, remove it, and "fill out" every + * level from the ancestor down to @p id's parent, inserting the + * siblings not on the path. After fill-out @p id and its + * descendants are uncovered, every other branch remains covered. + */ + void toggleSparseSelection( + DiffSection section, + std::string const& id, + bool nowSelected, + std::unordered_set& set + ) + { + if (nowSelected) + { + if (sparseAnyAncestorInSet(id, set)) + return; // already effectively selected + const std::string prefix = id + "/"; + std::vector toRemove; + for (auto const& key : set) { - SyncItem item{}; - item.action = actionToSyncItemAction(diffEntry.action); - if (diffEntry.local) - item.localItem = toFileExplorerItem(std::move(*diffEntry.local), diffEntry.relKey); - if (diffEntry.remote) - item.remoteItem = toFileExplorerItem(std::move(*diffEntry.remote), diffEntry.relKey); - item.relKey = std::move(diffEntry.relKey); - result.push_back(std::move(item)); + if (key.size() > prefix.size() && key.starts_with(prefix)) + toRemove.push_back(key); } - return result; - }; + for (auto const& key : toRemove) + set.erase(key); + set.insert(id); + collapseUpSparse(section, id, set); + return; + } - auto uploads = convert(std::move(diff.uploads)); - auto downloads = convert(std::move(diff.downloads)); - auto deletes = convert(std::move(diff.deletes)); - - // Auto-toggle a section's collapsed state on N↔0 transitions so the - // user isn't left looking at an empty open section after changing - // direction/actions, and isn't surprised by a hidden newly-populated - // section either. Only zero-crossings flip — manual collapses while - // a section stays populated are preserved. - auto reconcileCollapse = [](Nui::Observed& collapsed, - std::size_t prevCount, - std::size_t newCount) { - if (prevCount > 0 && newCount == 0) - collapsed = true; - else if (prevCount == 0 && newCount > 0) - collapsed = false; - }; - reconcileCollapse(uploadCollapsed_, uploadItems_.value().size(), uploads.size()); - reconcileCollapse(downloadCollapsed_, downloadItems_.value().size(), downloads.size()); - reconcileCollapse(deleteCollapsed_, deleteItems_.value().size(), deletes.size()); - - uploadItems_ = std::move(uploads); - downloadItems_ = std::move(downloads); - deleteItems_ = std::move(deletes); - resetSelectionAllChecked(); - refreshTrees(); + // Uncheck. + if (set.contains(id)) + { + set.erase(id); + return; + } + const auto covering = sparseFindCoveringAncestor(id, set); + if (covering.empty()) + { + // No ancestor in set — any "checked" state here was contributed by + // descendants. Remove them all so the row flips to Unchecked. + const std::string prefix = id + "/"; + std::vector toRemove; + for (auto const& key : set) + { + if (key.size() > prefix.size() && key.starts_with(prefix)) + toRemove.push_back(key); + } + for (auto const& key : toRemove) + set.erase(key); + return; + } + set.erase(covering); + fillOutSparse(section, covering, id, set); } - /** @brief Resets each tree's selection set to contain every current tree-leaf - * relKey — the "all checked" default users expect after a recompare. + /** @brief Walks from @p id up toward the root, replacing fully-selected + * sibling groups with their parent. Stops at the top level — + * root-level entries are the sparsest "all selected" representation. */ - void resetSelectionAllChecked(); + void collapseUpSparse( + DiffSection section, + std::string id, + std::unordered_set& set + ) + { + auto& tree = treeFor(section); + while (!id.empty()) + { + const auto parent = sparseParentOf(id); + if (parent.empty()) + return; // don't collapse top-level into synthetic root + const auto siblings = tree.childrenOf(parent); + if (siblings.empty()) + return; + const bool allIn = + std::all_of(siblings.begin(), siblings.end(), [&](auto const& s) { + return set.contains(s); + }); + if (!allIn) + return; + for (auto const& s : siblings) + set.erase(s); + set.insert(parent); + id = parent; + } + } - /** @brief Enqueues a single item from one of the three diff lists at priority. - * - * @param list The observed list the item belongs to (upload, download or delete). - * @param index Index of the item inside @p list. - */ - /** @brief True iff the SyncItem refers to a directory entry on either side. - * Directories only appear in the diff lists when missing on the - * opposite side; when the user has disabled recursion we must NOT - * dispatch them as bulk transfers (which would walk the local tree - * and could overwrite remote files the user never saw in the diff). + /** @brief Expands a covering ancestor into per-sibling selections along the + * path to @p target. At every level between @p ancestor and @p + * target, inserts every sibling NOT on the path; descends into the + * path child. Precondition: @p ancestor is a strict prefix of + * @p target in relKey terms. */ - static bool isDirectoryItem(SyncItem const& itm) + void fillOutSparse( + DiffSection section, + std::string const& ancestor, + std::string const& target, + std::unordered_set& set + ) { - if (itm.localItem && itm.localItem->type == SharedData::FileType::Directory) - return true; - if (itm.remoteItem && itm.remoteItem->type == SharedData::FileType::Directory) - return true; - return false; + auto& tree = treeFor(section); + std::string current = ancestor; + while (current != target) + { + const std::string remainder = + current.empty() ? target : target.substr(current.size() + 1); + const auto slash = remainder.find('/'); + const std::string nextSeg = + slash == std::string::npos ? remainder : remainder.substr(0, slash); + const std::string nextChildId = + current.empty() ? nextSeg : current + "/" + nextSeg; + + const auto siblings = tree.childrenOf(current); + for (auto const& sib : siblings) + { + if (sib != nextChildId) + set.insert(sib); + } + current = nextChildId; + } } - void enqueueSingle(Nui::Observed>& list, std::size_t index) + ScriptNuiComponents::Tree::ChildrenLoader makeChildrenLoader(DiffSection section) { - auto items = list.value(); - if (index >= items.size()) - return; - - auto progress = std::make_shared>(0.0); - items[index].progress = progress; - const auto itemCopy = items[index]; - list = std::move(items); - refreshTrees(); - Nui::globalEventContext.executeActiveEventsImmediately(); - - auto onComplete = [this, progress](std::optional const& opId, std::string const&) - { - if (!opId) + return [this, section]( + ScriptNuiComponents::Tree::NodeId const& parentId, + std::function)> resolve, + std::function reject) { + if (!provider_) { - *progress = -1.0; - Nui::globalEventContext.executeActiveEventsImmediately(); + reject("no provider"); return; } - operationQueue_->addTransferProgressCallback( - *opId, - [progress](double fraction) { - *progress = fraction; - Nui::globalEventContext.executeActiveEventsImmediately(); - } - ); - operationQueue_->addCompletionCallback( - *opId, - [progress](bool) { - *progress = 1.1; - Nui::globalEventContext.executeActiveEventsImmediately(); + provider_->loadChildren( + section, + parentId, + [resolve](std::vector nodes) { + std::vector result; + result.reserve(nodes.size()); + for (auto& node : nodes) + result.push_back(toTreeNode(std::move(node))); + resolve(std::move(result)); + }, + [reject](std::string const& msg) { + reject(msg); } ); }; + } - // In non-recursive mode a directory entry only ever means "create the - // empty dir on the other side" — we have not scanned its children, so - // a bulk transfer here would walk the local tree and could clobber - // remote files the user never saw in the diff. - const bool dirOnly = !recursive_.value() && isDirectoryItem(itemCopy); - auto onDirCreated = [progress](bool success, std::string const&) { - *progress = success ? 1.1 : -1.0; - Nui::globalEventContext.executeActiveEventsImmediately(); + static ScriptNuiComponents::Tree::Node toTreeNode(DiffTreeNode node) + { + const auto id = node.relKey; + const bool isDir = node.isDirectory; + return ScriptNuiComponents::Tree::Node{ + .id = id, + .kind = isDir ? ScriptNuiComponents::Tree::NodeKind::Directory + : ScriptNuiComponents::Tree::NodeKind::Leaf, + .children = {}, + .hasChildren = node.directChildCount > 0, + .userData = std::move(node), + .icon = std::nullopt, + .initiallyExpanded = false, + .selectable = true, }; + } + + /** @brief Build the row renderer for one section. Reads the DiffTreeNode out + * of the tree's userData and paints the per-row gradient using an + * on-demand progress observer (see @ref progressMapFor). + */ + ScriptNuiComponents::Tree::RowContentRenderer makeTreeRowRenderer(DiffSection section); - switch (itemCopy.action) + ScriptNuiComponents::Tree::RowAttributeProvider makeTreeRowAttributes() + { + return {}; + } + + /** + * @brief Applies the new diff summary and (re)seeds the three trees. The + * @p collapseZero flag controls whether sections whose counts cross + * the 0-boundary auto-toggle their collapse state (only on initial + * open / on settings-triggered recompute). + */ + void applySummaryAndReseed(SharedData::Sync::DiffSummary summary, bool collapseZero) + { + if (collapseZero) { - case SyncItemAction::Upload: - { - if (dirOnly) - { - operationQueue_->createRemoteDirectory( - remotePath_ / itemCopy.relKey, onDirCreated - ); - } - else if (itemCopy.localItem && itemCopy.remoteItem) - { - operationQueue_->enqueueUpload( - *itemCopy.remoteItem, *itemCopy.localItem, onComplete, true, false, /*createMissingDirs=*/true, - SharedData::OperationMode::PriorityQueued - ); - } - else if (itemCopy.localItem) - { - SharedData::DirectoryEntry remoteStub = *itemCopy.localItem; - remoteStub.path = itemCopy.localItem->path; - remoteStub.fullPath = remotePath_ / itemCopy.localItem->path; - operationQueue_->enqueueUpload( - NuiFileExplorer::Item{remoteStub}, *itemCopy.localItem, onComplete, true, false, - /*createMissingDirs=*/true, SharedData::OperationMode::PriorityQueued - ); - } - break; - } - case SyncItemAction::Download: - { - if (dirOnly) - { - operationQueue_->createLocalDirectory( - localPath_ / itemCopy.relKey, onDirCreated - ); - } - else if (itemCopy.localItem && itemCopy.remoteItem) - { - operationQueue_->enqueueDownload( - *itemCopy.remoteItem, *itemCopy.localItem, onComplete, true, false, /*createMissingDirs=*/true, - SharedData::OperationMode::PriorityQueued - ); - } - else if (itemCopy.remoteItem) - { - SharedData::DirectoryEntry localStub = *itemCopy.remoteItem; - localStub.path = itemCopy.remoteItem->path; - localStub.fullPath = localPath_ / itemCopy.remoteItem->path; - operationQueue_->enqueueDownload( - *itemCopy.remoteItem, NuiFileExplorer::Item{localStub}, onComplete, true, false, - /*createMissingDirs=*/true, SharedData::OperationMode::PriorityQueued - ); - } - break; - } - case SyncItemAction::DeleteLocal: - case SyncItemAction::DeleteRemote: - { - std::vector paths; - if (itemCopy.action == SyncItemAction::DeleteRemote && itemCopy.remoteItem) - paths.push_back(itemCopy.remoteItem->fullPath); - else if (itemCopy.action == SyncItemAction::DeleteLocal && itemCopy.localItem) - paths.push_back(itemCopy.localItem->fullPath); - if (!paths.empty()) - { - operationQueue_->enqueueDelete( - paths, recursive_.value(), - [progress](auto const& opIds, std::string const&) { - *progress = opIds ? 1.1 : -1.0; - Nui::globalEventContext.executeActiveEventsImmediately(); - }, - SharedData::OperationMode::PriorityQueued - ); - } - break; - } + const auto prev = summary_.value(); + auto reconcile = [](Nui::Observed& collapsed, + std::uint64_t prevCount, + std::uint64_t newCount) { + if (prevCount > 0 && newCount == 0) + collapsed = true; + else if (prevCount == 0 && newCount > 0) + collapsed = false; + }; + reconcile(uploadCollapsed_, prev.uploads.itemCount, summary.uploads.itemCount); + reconcile(downloadCollapsed_, prev.downloads.itemCount, summary.downloads.itemCount); + reconcile(deleteCollapsed_, prev.deletes.itemCount, summary.deletes.itemCount); } + + summary_ = summary; + + // Clear per-section progress — rows are about to be re-fetched and any + // in-flight row observers won't match the new generation anyway. + uploadProgress_.clear(); + downloadProgress_.clear(); + deleteProgress_.clear(); + + seedRootNodes(); } - /** @brief Computes the minimal set of item indices to enqueue from @p items - * given the user's @p selected leaf set. - * - * Directory-role items (one side is absent, the present side is a Directory) - * cause the backend to recursively transfer/delete the whole subtree. When - * every descendant item is also selected, we emit just the directory and - * skip its descendants — the former behaviour of emitting the directory - * AND every descendant caused each leaf to be transferred N times, where N - * is its depth below the top-most directory-role ancestor. - * - * When a directory has any deselected descendant, we cannot use its bulk - * operation (it would act on items the user excluded), so we recurse and - * emit only selected tree-leaves. Intermediate directory-role items with - * partial selection are not emitted themselves: for Upload/Download the - * leaf enqueues use createMissingDirectories; for Delete the remaining - * non-empty dir is an acceptable no-op. + /** @brief Fetches root-level children for every section and, once each + * reply lands, hands them to @ref Tree::setRoots and seeds that + * section's sparse selection set to the root relKeys (= "all + * selected" in sparsest form). */ - static std::vector minimizeEnqueueIndices( - std::vector const& items, - std::unordered_set const& selected) + void seedRootNodes() { - const auto isBulkDir = [](SyncItem const& item) { - switch (item.action) - { - case SyncItemAction::Download: - return !item.localItem && item.remoteItem && - item.remoteItem->type == SharedData::FileType::Directory; - case SyncItemAction::Upload: - return !item.remoteItem && item.localItem && - item.localItem->type == SharedData::FileType::Directory; - case SyncItemAction::DeleteLocal: - return item.localItem && - item.localItem->type == SharedData::FileType::Directory; - case SyncItemAction::DeleteRemote: - return item.remoteItem && - item.remoteItem->type == SharedData::FileType::Directory; - } - return false; + if (!provider_) + return; + auto seedOne = [this](DiffSection section) { + provider_->loadChildren( + section, + std::string{}, + [this, section](std::vector nodes) { + std::vector roots; + roots.reserve(nodes.size()); + for (auto& node : nodes) + roots.push_back(toTreeNode(std::move(node))); + treeFor(section).setRoots(std::move(roots)); + auto selectedPtr = selectionFor(section); + auto& set = selectedPtr->value(); + set.clear(); + for (auto const& rootId : treeFor(section).childrenOf(std::string{})) + set.insert(rootId); + selectedPtr->modify(); + Nui::globalEventContext.executeActiveEventsImmediately(); + }, + [](std::string const& msg) { + Log::error("Sync seedRootNodes failed: {}", msg); + } + ); }; + seedOne(DiffSection::Upload); + seedOne(DiffSection::Download); + seedOne(DiffSection::Delete); + } - std::vector views; - views.reserve(items.size()); - for (auto const& item : items) - views.push_back({.relKey = item.relKey, .isBulkDir = isBulkDir(item)}); - return SharedData::Sync::minimizeEnqueueIndices(views, selected); + SharedData::Sync::DiffOptions buildDiffOptions() const + { + return SharedData::Sync::DiffOptions{ + .direction = direction_, + .actionUpload = actionUpload_.value(), + .actionDownload = actionDownload_.value(), + .actionDelete = actionDelete_.value(), + .recursive = recursive_.value(), + .ignoreHidden = ignoreHidden_.value(), + }; } - void enqueueOperations() + /** @brief Triggered on any settings change — runs a fresh backend diff and + * reseeds the trees. Collapses sections that zero out, expands ones + * that gain rows. + */ + void recomputeDiff() { - const auto uploadIndices = minimizeEnqueueIndices(uploadItems_.value(), uploadSelected_->value()); - const auto downloadIndices = minimizeEnqueueIndices(downloadItems_.value(), downloadSelected_->value()); - const auto deleteIndices = minimizeEnqueueIndices(deleteItems_.value(), deleteSelected_->value()); - - // Each selected row gets its own progress observer (propagated to every - // descendant SyncItem so subtree rows reflect the ancestor's progress). - // Observers must be in place before any RPC callback fires. - auto assignProgress = [](std::vector& items, std::vector const& emitted) { - for (auto idx : emitted) - { - auto prog = std::make_shared>(0.0); - items[idx].progress = prog; - const std::string prefix = items[idx].relKey + "/"; - for (auto& other : items) - { - if (other.relKey.size() > prefix.size() && other.relKey.starts_with(prefix)) - other.progress = prog; - } + if (!provider_) + return; + provider_->recompute( + buildDiffOptions(), + [this](SharedData::Sync::DiffSummary summary) { + applySummaryAndReseed(std::move(summary), /*collapseZero=*/true); + Nui::globalEventContext.executeActiveEventsImmediately(); } - }; + ); + } + /** @brief Allocates a shared progress observer and propagates it to @p relKey + * plus every currently-loaded descendant row in the tree. Used for + * bulk-directory enqueue where the whole subtree animates off a + * single backend progress stream. + */ + ProgressObserver installProgressFor(DiffSection section, std::string const& relKey) + { + auto& progMap = progressMapFor(section); + auto& entry = progMap[relKey]; + if (!entry) + entry = std::make_shared>(0.0); + // Propagate to any already-loaded descendants so their rows share the + // same gradient. Descendants loaded AFTER enqueue inherit automatically + // via the row renderer's prefix lookup below. + const std::string prefix = relKey + "/"; + for (auto& [key, obs] : progMap) { - auto uploads = uploadItems_.value(); - assignProgress(uploads, uploadIndices); - uploadItems_ = std::move(uploads); - } - { - auto downloads = downloadItems_.value(); - assignProgress(downloads, downloadIndices); - downloadItems_ = std::move(downloads); + if (key.size() > prefix.size() && key.starts_with(prefix)) + obs = entry; } + return entry; + } + + /** @brief Looks up the progress observer that applies to @p relKey — its own + * if present, otherwise the nearest ancestor's. Returns nullptr when + * the row has no enqueued transfer affecting it. + */ + ProgressObserver progressForRow(DiffSection section, std::string const& relKey) const + { + auto const& progMap = + section == DiffSection::Upload ? uploadProgress_ : + section == DiffSection::Download ? downloadProgress_ : + deleteProgress_; + if (auto iter = progMap.find(relKey); iter != progMap.end()) + return iter->second; + // Walk ancestor prefixes. + std::string candidate = relKey; + while (!candidate.empty()) { - auto deletes = deleteItems_.value(); - assignProgress(deletes, deleteIndices); - deleteItems_ = std::move(deletes); + const auto slash = candidate.rfind('/'); + if (slash == std::string::npos) + candidate.clear(); + else + candidate.resize(slash); + if (auto iter = progMap.find(candidate); iter != progMap.end()) + return iter->second; } + return nullptr; + } - const bool nonRecursive = !recursive_.value(); - - refreshTrees(); + /** + * @brief Enqueues a single row. The row click path has the full DiffTreeNode + * in hand (via Tree row userData) so we don't need a provider round-trip. + */ + void enqueueSingleNode(DiffSection section, DiffTreeNode const& node) + { + auto progress = installProgressFor(section, node.relKey); + ++progressEpoch_.value(); + progressEpoch_.modify(); Nui::globalEventContext.executeActiveEventsImmediately(); - auto onDirCreatedFor = [](std::shared_ptr> prog) { - return [prog](bool success, std::string const&) { - *prog = success ? 1.1 : -1.0; + const bool dirOnly = !recursive_.value() && node.isDirectory; + auto onDirCreated = [progress](bool success, std::string const&) { + *progress = success ? 1.1 : -1.0; + Nui::globalEventContext.executeActiveEventsImmediately(); + }; + auto onComplete = [this, progress](std::optional const& opId, std::string const&) { + if (!opId) + { + *progress = -1.0; Nui::globalEventContext.executeActiveEventsImmediately(); - }; + return; + } + operationQueue_->addTransferProgressCallback(*opId, [progress](double fraction) { + *progress = fraction; + Nui::globalEventContext.executeActiveEventsImmediately(); + }); + operationQueue_->addCompletionCallback(*opId, [progress](bool) { + *progress = 1.1; + Nui::globalEventContext.executeActiveEventsImmediately(); + }); }; - // Live per-row progress for bulk transfers is reconstructed from - // BulkProgress events (aggregated per-batch, carrying currentFile + - // currentFileBytes/currentFileTotalBytes) plus per-entry completion - // callbacks. We index progress observers by the canonical src path - // so BulkProgress.currentFile can be resolved in O(1). - using ProgressPtr = std::shared_ptr>; - auto hookBulkTransfer = [this]( - std::vector entries, - std::vector observers, - bool isUpload, - std::string const& kind - ) { - // The backend collapses file entries into a single prescanned - // BulkUpload/BulkDownload (directories are split out into their - // own per-opId operations). BulkProgress.fileCurrentIndex thus - // indexes the file-only list in the same order as our non-dir - // frontend entries — build a lookup from that file-index back - // to the frontend entry index so we can drive per-row progress - // and per-file green-highlighting without path matching (which - // failed for uploads because currentFile carries the remote - // dst, not the src we were keyed by). - auto fileEntryIndices = std::make_shared>(); - fileEntryIndices->reserve(entries.size()); - for (std::size_t idx = 0; idx < entries.size(); ++idx) - if (!entries[idx].isDirectory) - fileEntryIndices->push_back(idx); - - auto entryIsDir = std::make_shared>(); - entryIsDir->reserve(entries.size()); - for (auto const& entry : entries) - entryIsDir->push_back(entry.isDirectory); - auto observersShared = std::make_shared>(std::move(observers)); - - // Flip observers for all file entries strictly before backend - // file-index `upTo` to the completed (green) state, unless they - // are already in a terminal state. - auto markFilesCompletedBefore = - [observersShared, fileEntryIndices](std::uint64_t upTo) { - const auto limit = std::min(upTo, fileEntryIndices->size()); - for (std::uint64_t pos = 0; pos < limit; ++pos) - { - auto& obs = (*observersShared)[(*fileEntryIndices)[pos]]; - if (!obs) - continue; - const double value = obs->value(); - if (value >= 1.0 || value < 0.0) - continue; - *obs = 1.1; - } - }; - - auto onBulkProgress = - [observersShared, fileEntryIndices, markFilesCompletedBefore](SharedData::BulkProgress const& prog) { - markFilesCompletedBefore(prog.fileCurrentIndex); - if (prog.fileCurrentIndex >= fileEntryIndices->size()) - { - Nui::globalEventContext.executeActiveEventsImmediately(); - return; - } - auto& obs = (*observersShared)[(*fileEntryIndices)[prog.fileCurrentIndex]]; - if (obs && prog.currentFileTotalBytes > 0) - { - *obs = static_cast(prog.currentFileBytes) - / static_cast(prog.currentFileTotalBytes); - } - Nui::globalEventContext.executeActiveEventsImmediately(); - }; + const auto localAbs = localPath_ / node.relKey; + const auto remoteAbs = remotePath_ / node.relKey; - auto onEnqueued = [this, - onBulkProgress = std::move(onBulkProgress), - observersShared, - entryIsDir](std::vector const& opIds) { - if (opIds.empty()) + switch (node.action) + { + case SharedData::Sync::Action::Upload: + { + if (dirOnly) + { + operationQueue_->createRemoteDirectory(remoteAbs, onDirCreated); return; - // The frontend reserves entries.size() + 1 opIds — the trailing - // one is the aggregate bulk-card id that the backend keys both - // BulkProgress and the aggregate OperationCompleted to. The - // per-entry opIds[0..N-1] only drive directory sub-operations. - operationQueue_->addBulkProgressCallback(opIds.back(), onBulkProgress); - - operationQueue_->addCompletionCallback( - opIds.back(), - [observersShared, entryIsDir](bool success) { - for (std::size_t idx = 0; idx < observersShared->size(); ++idx) - { - if (idx < entryIsDir->size() && (*entryIsDir)[idx]) - continue; // dir entries flip via their own opId callback below - auto& obs = (*observersShared)[idx]; - if (obs) - *obs = success ? 1.1 : -1.0; - } - Nui::globalEventContext.executeActiveEventsImmediately(); - } + } + SharedData::DirectoryEntry localEntry{}; + localEntry.path = std::filesystem::path{node.relKey}; + localEntry.fullPath = localAbs; + localEntry.size = node.localSize; + localEntry.mtime = node.localMtime; + localEntry.type = node.isDirectory ? SharedData::FileType::Directory : SharedData::FileType::Regular; + SharedData::DirectoryEntry remoteEntry = localEntry; + remoteEntry.fullPath = remoteAbs; + operationQueue_->enqueueUpload( + NuiFileExplorer::Item{remoteEntry}, + NuiFileExplorer::Item{localEntry}, + onComplete, + /*allowOverwrite=*/true, + /*insertRefresh=*/false, + /*createMissingDirs=*/true, + SharedData::OperationMode::PriorityQueued ); - for (std::size_t idx = 0; idx < opIds.size() && idx < entryIsDir->size(); ++idx) + return; + } + case SharedData::Sync::Action::Download: + { + if (dirOnly) { - if (!(*entryIsDir)[idx]) - continue; - auto observer = (idx < observersShared->size()) ? (*observersShared)[idx] : nullptr; - if (!observer) - continue; - operationQueue_->addCompletionCallback( - opIds[idx], - [observer](bool success) { - *observer = success ? 1.1 : -1.0; - Nui::globalEventContext.executeActiveEventsImmediately(); - } - ); + operationQueue_->createLocalDirectory(localAbs, onDirCreated); + return; } - }; + SharedData::DirectoryEntry remoteEntry{}; + remoteEntry.path = std::filesystem::path{node.relKey}; + remoteEntry.fullPath = remoteAbs; + remoteEntry.size = node.remoteSize; + remoteEntry.mtime = node.remoteMtime; + remoteEntry.type = node.isDirectory ? SharedData::FileType::Directory : SharedData::FileType::Regular; + SharedData::DirectoryEntry localEntry = remoteEntry; + localEntry.fullPath = localAbs; + operationQueue_->enqueueDownload( + NuiFileExplorer::Item{remoteEntry}, + NuiFileExplorer::Item{localEntry}, + onComplete, + /*allowOverwrite=*/true, + /*insertRefresh=*/false, + /*createMissingDirs=*/true, + SharedData::OperationMode::PriorityQueued + ); + return; + } + case SharedData::Sync::Action::DeleteLocal: + case SharedData::Sync::Action::DeleteRemote: + { + std::vector paths; + paths.push_back(node.action == SharedData::Sync::Action::DeleteRemote ? remoteAbs : localAbs); + operationQueue_->enqueueDelete( + paths, + recursive_.value(), + [progress](auto const& opIds, std::string const&) { + *progress = opIds ? 1.1 : -1.0; + Nui::globalEventContext.executeActiveEventsImmediately(); + }, + SharedData::OperationMode::PriorityQueued + ); + return; + } + } + } + + void enqueueOperations() + { + if (!provider_) + return; + enqueueOneSection(DiffSection::Upload); + enqueueOneSection(DiffSection::Download); + enqueueOneSection(DiffSection::Delete); + } - auto onBulkAck = [kind](bool success, std::string const& info) { - if (!success) - Log::error("Sync bulk {} failed: {}", kind, info); - }; + /** @brief Asks the backend to collapse the section's selected relKeys into a + * minimal enqueue plan, then dispatches each plan entry. Bulk files + * are batched into a BulkUpload/BulkDownload; directory-only entries + * (non-recursive mode) become single createRemote/LocalDirectory + * calls; deletes go to enqueueBulkDelete. + */ + void enqueueOneSection(DiffSection section) + { + auto const& selectedSet = selectionFor(section)->value(); + std::vector selected{selectedSet.begin(), selectedSet.end()}; + if (selected.empty()) + return; - if (isUpload) - { - operationQueue_->enqueueBulkUpload( - std::move(entries), /*allowOverwrite*/ true, /*insertRefresh*/ false, - SharedData::OperationMode::Queued, - /*onEachComplete*/ {}, std::move(onBulkAck), std::move(onEnqueued) - ); - } - else - { - operationQueue_->enqueueBulkDownload( - std::move(entries), /*allowOverwrite*/ true, /*insertRefresh*/ false, - SharedData::OperationMode::Queued, - /*onEachComplete*/ {}, std::move(onBulkAck), std::move(onEnqueued) - ); + provider_->buildEnqueuePlan( + section, + std::move(selected), + [this, section](std::vector plan) { + dispatchPlan(section, std::move(plan)); + Nui::globalEventContext.executeActiveEventsImmediately(); + }, + [](std::string const& msg) { + Log::error("buildSyncEnqueuePlan failed: {}", msg); } + ); + } + + void dispatchPlan(DiffSection section, std::vector plan) + { + if (plan.empty()) + return; + + using ProgressObserverVec = std::vector; + + const bool nonRecursive = !recursive_.value(); + auto onDirCreatedFor = [](ProgressObserver prog) { + return [prog](bool success, std::string const&) { + *prog = success ? 1.1 : -1.0; + Nui::globalEventContext.executeActiveEventsImmediately(); + }; }; std::vector uploadEntries; - std::vector uploadObservers; - auto const& uploadsSnap = uploadItems_.value(); - for (auto idx : uploadIndices) + ProgressObserverVec uploadObservers; + std::vector downloadEntries; + ProgressObserverVec downloadObservers; + std::vector deleteEntries; + ProgressObserverVec deleteObservers; + + for (auto& entry : plan) { - auto const& itm = uploadsSnap[idx]; - if (nonRecursive && isDirectoryItem(itm)) - { - operationQueue_->createRemoteDirectory( - remotePath_ / itm.relKey, onDirCreatedFor(itm.progress) - ); - continue; - } - if (itm.localItem && itm.remoteItem) - { - uploadEntries.push_back(SharedData::BulkAddEntry{ - .src = !itm.localItem->fullPath.empty() ? itm.localItem->fullPath : itm.localItem->path, - .dst = !itm.remoteItem->fullPath.empty() ? itm.remoteItem->fullPath : itm.remoteItem->path, - .sizeBytes = itm.localItem->size, - .isDirectory = itm.localItem->isDirectory(), - }); - uploadObservers.push_back(itm.progress); - } - else if (itm.localItem) + auto progress = installProgressFor(section, entry.relKey); + + switch (entry.action) { - uploadEntries.push_back(SharedData::BulkAddEntry{ - .src = !itm.localItem->fullPath.empty() ? itm.localItem->fullPath : itm.localItem->path, - .dst = remotePath_ / itm.localItem->path, - .sizeBytes = itm.localItem->size, - .isDirectory = itm.localItem->isDirectory(), - }); - uploadObservers.push_back(itm.progress); + case SharedData::Sync::Action::Upload: + { + if (nonRecursive && entry.isDirectory) + { + operationQueue_->createRemoteDirectory( + std::filesystem::path{entry.remoteAbsPath}, onDirCreatedFor(progress) + ); + continue; + } + uploadEntries.push_back(SharedData::BulkAddEntry{ + .src = entry.localAbsPath, + .dst = entry.remoteAbsPath, + .sizeBytes = entry.sizeBytes, + .isDirectory = entry.isDirectory, + .mtime = entry.mtime, + .mtimeNsec = entry.mtimeNsec, + }); + uploadObservers.push_back(progress); + break; + } + case SharedData::Sync::Action::Download: + { + if (nonRecursive && entry.isDirectory) + { + operationQueue_->createLocalDirectory( + std::filesystem::path{entry.localAbsPath}, onDirCreatedFor(progress) + ); + continue; + } + downloadEntries.push_back(SharedData::BulkAddEntry{ + .src = entry.remoteAbsPath, + .dst = entry.localAbsPath, + .sizeBytes = entry.sizeBytes, + .isDirectory = entry.isDirectory, + .mtime = entry.mtime, + .mtimeNsec = entry.mtimeNsec, + }); + downloadObservers.push_back(progress); + break; + } + case SharedData::Sync::Action::DeleteLocal: + { + deleteEntries.push_back(SharedData::BulkAddEntry{ + .src = entry.localAbsPath, + .dst = {}, + .sizeBytes = 0, + .isDirectory = entry.isDirectory, + }); + deleteObservers.push_back(progress); + break; + } + case SharedData::Sync::Action::DeleteRemote: + { + deleteEntries.push_back(SharedData::BulkAddEntry{ + .src = entry.remoteAbsPath, + .dst = {}, + .sizeBytes = 0, + .isDirectory = entry.isDirectory, + }); + deleteObservers.push_back(progress); + break; + } } } + + ++progressEpoch_.value(); + progressEpoch_.modify(); + if (!uploadEntries.empty()) hookBulkTransfer(std::move(uploadEntries), std::move(uploadObservers), /*isUpload=*/true, "upload"); - - std::vector downloadEntries; - std::vector downloadObservers; - auto const& downloadsSnap = downloadItems_.value(); - for (auto idx : downloadIndices) - { - auto const& itm = downloadsSnap[idx]; - if (nonRecursive && isDirectoryItem(itm)) - { - operationQueue_->createLocalDirectory( - localPath_ / itm.relKey, onDirCreatedFor(itm.progress) - ); - continue; - } - if (itm.localItem && itm.remoteItem) - { - downloadEntries.push_back(SharedData::BulkAddEntry{ - .src = !itm.remoteItem->fullPath.empty() ? itm.remoteItem->fullPath : itm.remoteItem->path, - .dst = !itm.localItem->fullPath.empty() ? itm.localItem->fullPath : itm.localItem->path, - .sizeBytes = itm.remoteItem->size, - .isDirectory = itm.remoteItem->isDirectory(), - .mtime = itm.remoteItem->mtime, - .mtimeNsec = itm.remoteItem->mtimeNsec, - }); - downloadObservers.push_back(itm.progress); - } - else if (itm.remoteItem) - { - downloadEntries.push_back(SharedData::BulkAddEntry{ - .src = !itm.remoteItem->fullPath.empty() ? itm.remoteItem->fullPath : itm.remoteItem->path, - .dst = localPath_ / itm.remoteItem->path, - .sizeBytes = itm.remoteItem->size, - .isDirectory = itm.remoteItem->isDirectory(), - .mtime = itm.remoteItem->mtime, - .mtimeNsec = itm.remoteItem->mtimeNsec, - }); - downloadObservers.push_back(itm.progress); - } - } if (!downloadEntries.empty()) hookBulkTransfer(std::move(downloadEntries), std::move(downloadObservers), /*isUpload=*/false, "download"); - auto const& deletesSnap = deleteItems_.value(); - std::vector deleteEntries; - std::vector deleteObservers; - deleteEntries.reserve(deleteIndices.size()); - deleteObservers.reserve(deleteIndices.size()); - for (auto idx : deleteIndices) - { - auto const& itm = deletesSnap[idx]; - if (itm.action == SyncItemAction::DeleteRemote && itm.remoteItem) - { - deleteEntries.push_back(SharedData::BulkAddEntry{ - .src = itm.remoteItem->fullPath, - .dst = {}, - .sizeBytes = 0, - .isDirectory = itm.remoteItem->isDirectory(), - }); - deleteObservers.push_back(itm.progress); - } - else if (itm.action == SyncItemAction::DeleteLocal && itm.localItem) - { - deleteEntries.push_back(SharedData::BulkAddEntry{ - .src = itm.localItem->fullPath, - .dst = {}, - .sizeBytes = 0, - .isDirectory = itm.localItem->isDirectory(), - }); - deleteObservers.push_back(itm.progress); - } - } if (!deleteEntries.empty()) { - // Bulk delete emits BulkDeleteProgress (not BulkProgress) and only - // signals completion once for the whole batch — flip all delete - // rows together when onBulkComplete fires. - auto observersShared = std::make_shared>(std::move(deleteObservers)); + auto observersShared = std::make_shared(std::move(deleteObservers)); operationQueue_->enqueueBulkDelete( std::move(deleteEntries), - /*insertRefresh*/ false, + /*insertRefresh=*/false, SharedData::OperationMode::Queued, [observersShared](bool success) { for (auto& obs : *observersShared) @@ -870,126 +1035,232 @@ struct SyncDialog::Implementation ); } } -}; -// ---- Tree integration ------------------------------------------------------ - -namespace -{ - /** @brief Pull a SyncItem out of a Tree row's userData (pointer type is - * expected, may be null — callers must check). + /** @brief Shared wiring between BulkUpload and BulkDownload. Tracks per-file + * progress via the backend's BulkProgress event stream and marks + * completed rows green. */ - SyncItem const* userDataAsSyncItem(std::any const& userData) + void hookBulkTransfer( + std::vector entries, + std::vector observers, + bool isUpload, + std::string const& kind + ) { - auto const* ptr = std::any_cast(&userData); - return ptr ? *ptr : nullptr; + auto fileEntryIndices = std::make_shared>(); + fileEntryIndices->reserve(entries.size()); + for (std::size_t idx = 0; idx < entries.size(); ++idx) + if (!entries[idx].isDirectory) + fileEntryIndices->push_back(idx); + + auto entryIsDir = std::make_shared>(); + entryIsDir->reserve(entries.size()); + for (auto const& entry : entries) + entryIsDir->push_back(entry.isDirectory); + auto observersShared = std::make_shared>(std::move(observers)); + + auto markFilesCompletedBefore = + [observersShared, fileEntryIndices](std::uint64_t upTo) { + const auto limit = std::min(upTo, fileEntryIndices->size()); + for (std::uint64_t pos = 0; pos < limit; ++pos) + { + auto& obs = (*observersShared)[(*fileEntryIndices)[pos]]; + if (!obs) + continue; + const double value = obs->value(); + if (value >= 1.0 || value < 0.0) + continue; + *obs = 1.1; + } + }; + + auto onBulkProgress = + [observersShared, fileEntryIndices, markFilesCompletedBefore](SharedData::BulkProgress const& prog) { + markFilesCompletedBefore(prog.fileCurrentIndex); + if (prog.fileCurrentIndex >= fileEntryIndices->size()) + { + Nui::globalEventContext.executeActiveEventsImmediately(); + return; + } + auto& obs = (*observersShared)[(*fileEntryIndices)[prog.fileCurrentIndex]]; + if (obs && prog.currentFileTotalBytes > 0) + { + *obs = static_cast(prog.currentFileBytes) + / static_cast(prog.currentFileTotalBytes); + } + Nui::globalEventContext.executeActiveEventsImmediately(); + }; + + auto onEnqueued = [this, + onBulkProgress = std::move(onBulkProgress), + observersShared, + entryIsDir](std::vector const& opIds) { + if (opIds.empty()) + return; + operationQueue_->addBulkProgressCallback(opIds.back(), onBulkProgress); + operationQueue_->addCompletionCallback( + opIds.back(), + [observersShared, entryIsDir](bool success) { + for (std::size_t idx = 0; idx < observersShared->size(); ++idx) + { + if (idx < entryIsDir->size() && (*entryIsDir)[idx]) + continue; + auto& obs = (*observersShared)[idx]; + if (obs) + *obs = success ? 1.1 : -1.0; + } + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ); + for (std::size_t idx = 0; idx < opIds.size() && idx < entryIsDir->size(); ++idx) + { + if (!(*entryIsDir)[idx]) + continue; + auto observer = (idx < observersShared->size()) ? (*observersShared)[idx] : nullptr; + if (!observer) + continue; + operationQueue_->addCompletionCallback( + opIds[idx], + [observer](bool success) { + *observer = success ? 1.1 : -1.0; + Nui::globalEventContext.executeActiveEventsImmediately(); + } + ); + } + }; + + auto onBulkAck = [kind](bool success, std::string const& info) { + if (!success) + Log::error("Sync bulk {} failed: {}", kind, info); + }; + + if (isUpload) + { + operationQueue_->enqueueBulkUpload( + std::move(entries), /*allowOverwrite*/ true, /*insertRefresh*/ false, + SharedData::OperationMode::Queued, + /*onEachComplete*/ {}, std::move(onBulkAck), std::move(onEnqueued) + ); + } + else + { + operationQueue_->enqueueBulkDownload( + std::move(entries), /*allowOverwrite*/ true, /*insertRefresh*/ false, + SharedData::OperationMode::Queued, + /*onEachComplete*/ {}, std::move(onBulkAck), std::move(onEnqueued) + ); + } } -} +}; -ScriptNuiComponents::Tree::RowContentRenderer -SyncDialog::Implementation::makeTreeRowRenderer(Nui::Observed>& listObs, bool mirrored) +// ---- Tree integration ------------------------------------------------------ + +ScriptNuiComponents::Tree::RowContentRenderer SyncDialog::Implementation::makeTreeRowRenderer(DiffSection section) { - // Where do content cells live for the current list? Static for Upload / - // Download (driven by the tree's `mirrored` flag), but the Delete list is - // one-sided and switches sides with the current direction: DeleteRemote - // → right, DeleteLocal → left. Synthetic directory rows read this at - // render time so their labels track the leaves underneath them. - auto contentOnRight = [mirrored, &listObs]() -> bool { - if (mirrored) - return true; - auto const& items = listObs.value(); - if (items.empty()) - return false; - return items.front().action == SyncItemAction::DeleteRemote; - }; - return [this, &listObs, contentOnRight](ScriptNuiComponents::Tree::RowContext const& ctx) -> Nui::ElementRenderer { + return [this, section](ScriptNuiComponents::Tree::RowContext const& ctx) -> Nui::ElementRenderer { using namespace Nui::Elements; using namespace Nui::Attributes; using Nui::Elements::div; using Nui::Elements::span; - SyncItem const* itemPtr = userDataAsSyncItem(ctx.userData); - if (!itemPtr) + DiffTreeNode const* nodePtr = userDataAsDiffTreeNode(ctx.userData); + if (!nodePtr) { - // Directory row — the tree provides the chevron + caller fills in a - // label based on the directory's basename derived from the NodeId. - const auto& fullKey = ctx.id; - std::string_view view{fullKey}; + // Fallback for any directory row the Tree synthesizes without userData + // (shouldn't happen now that the backend emits directory DiffTreeNodes, + // but the guard keeps rendering robust). + std::string_view view{ctx.id}; if (!view.empty() && view.back() == '/') view.remove_suffix(1); const auto slash = view.rfind('/'); const auto basename = (slash == std::string_view::npos) ? view : view.substr(slash + 1); - // Place the folder label on the same side as the leaf content in - // this list — otherwise a parent directory ends up floating on - // the opposite side of the row from its own children. - const bool labelRight = contentOnRight(); - auto labelCell = span{ - class_ = labelRight - ? "sync-diff-cell sync-diff-cell-right sync-diff-cell--directory" - : "sync-diff-cell sync-diff-cell--directory" - }(span{class_ = "name"}(std::string{basename})); - auto emptyCell = span{class_ = "sync-diff-cell sync-diff-cell--directory"}(); - if (labelRight) - { - return div{class_ = "sync-diff-row sync-diff-row--directory"}( - std::move(emptyCell), - div{class_ = "sync-diff-arrow"}(), - std::move(labelCell) - ); - } return div{class_ = "sync-diff-row sync-diff-row--directory"}( - std::move(labelCell), + span{class_ = "sync-diff-cell sync-diff-cell--directory"}( + span{class_ = "name"}(std::string{basename}) + ), div{class_ = "sync-diff-arrow"}(), - std::move(emptyCell) + span{class_ = "sync-diff-cell sync-diff-cell--directory"}() + ); + } + + DiffTreeNode const& node = *nodePtr; + + // Structural rows (synthesized ancestors of deep diffs) have no action + // and no transfer of their own — render a pure folder label with no arrow. + if (node.isStructural) + { + return div{class_ = "sync-diff-row sync-diff-row--directory"}( + span{class_ = "sync-diff-cell sync-diff-cell--directory"}( + span{class_ = "name"}(node.name) + ), + div{class_ = "sync-diff-arrow"}(), + span{class_ = "sync-diff-cell sync-diff-cell--directory"}() ); } - SyncItem const& itm = *itemPtr; std::string arrowClass; Nui::ElementRenderer arrowIcon = Nui::nil(); - switch (itm.action) + switch (node.action) { - case SyncItemAction::Upload: + case SharedData::Sync::Action::Upload: arrowClass = "upload"; arrowIcon = Ui5Icons::arrow_right(); break; - case SyncItemAction::Download: + case SharedData::Sync::Action::Download: arrowClass = "download"; arrowIcon = Ui5Icons::arrow_left(); break; - case SyncItemAction::DeleteLocal: - case SyncItemAction::DeleteRemote: + case SharedData::Sync::Action::DeleteLocal: + case SharedData::Sync::Action::DeleteRemote: arrowClass = "delete"; arrowIcon = Ui5Icons::delete_(); break; } - const std::string relKeyCopy = itm.relKey; - - // Single click target: the centre arrow / trashcan icon. Hover - // brightens and a press scales it down — see `sync_dialog.css`. - auto makeArrow = [&]() { + const DiffTreeNode nodeCopy = node; // captured into arrow click handler + auto makeArrow = [&, nodeCopy]() { return div{ class_ = fmt::format("sync-diff-arrow sync-diff-arrow--clickable {}", arrowClass), "title"_attr = language->get("syncDialog", "syncItemNowTitle"), - onClick = [this, &listObs, relKeyCopy](Nui::val event) { + onClick = [this, section, nodeCopy](Nui::val event) { event.call("stopPropagation"); - enqueueSingleByRelKey(listObs, relKeyCopy); + enqueueSingleNode(section, nodeCopy); }, }(std::move(arrowIcon)); }; - // Leaves render on the side that carries the data: local→left, - // remote→right. DeleteLocal/DeleteRemote inherit this via their - // populated side, so the Delete list naturally switches sides based - // on the current direction. - // Progress overlay is painted by `.sync-diff-row::before` via the - // `--sync-row-bg` CSS custom property set here. Keeping it on - // `.sync-diff-row`'s own `style` (not the outer tree row) avoids - // clobbering the tree's `--depth` var. - if (itm.progress) + // Directory rows (one-sided subtree emits): render a folder-style cell on + // whichever side holds the data. + if (node.isDirectory) + { + const bool labelRight = actionContentOnRight(node.action); + auto labelCell = span{ + class_ = labelRight + ? "sync-diff-cell sync-diff-cell-right sync-diff-cell--directory" + : "sync-diff-cell sync-diff-cell--directory" + }(span{class_ = "name"}(node.name)); + auto emptyCell = span{class_ = "sync-diff-cell sync-diff-cell--directory"}(); + if (labelRight) + { + return div{class_ = "sync-diff-row sync-diff-row--directory"}( + std::move(emptyCell), + makeArrow(), + std::move(labelCell) + ); + } + return div{class_ = "sync-diff-row sync-diff-row--directory"}( + std::move(labelCell), + makeArrow(), + std::move(emptyCell) + ); + } + + // Leaf row — progress gradient driven by whichever observer matches this + // relKey (own or nearest ancestor's). Repaint when progressEpoch_ moves. + auto progress = progressForRow(section, node.relKey); + if (progress) { - auto prog = itm.progress; + auto prog = progress; return div{ class_ = "sync-diff-row", style = Nui::observe(*prog).generate([prog]() -> std::string { @@ -1006,131 +1277,20 @@ SyncDialog::Implementation::makeTreeRowRenderer(Nui::Observed collectLeafRelKeys(std::vector const& list) - { - std::unordered_set leaves; - leaves.reserve(list.size()); - for (auto const& item : list) - { - const std::string prefix = item.relKey + "/"; - const bool hasChild = std::any_of(list.begin(), list.end(), [&](SyncItem const& other) { - return other.relKey.size() > prefix.size() && other.relKey.starts_with(prefix); - }); - if (!hasChild) - leaves.insert(item.relKey); - } - return leaves; - } -} - -void SyncDialog::Implementation::resetSelectionAllChecked() -{ - uploadSelected_->value() = collectLeafRelKeys(uploadItems_.value()); - uploadSelected_->modify(); - downloadSelected_->value() = collectLeafRelKeys(downloadItems_.value()); - downloadSelected_->modify(); - deleteSelected_->value() = collectLeafRelKeys(deleteItems_.value()); - deleteSelected_->modify(); -} - -void SyncDialog::Implementation::refreshTrees() -{ - namespace Snc = ScriptNuiComponents; - - auto fold = [](std::vector const& items) { - return Snc::foldByRelKey( - items, - [](SyncItem const& item) -> std::string_view { return item.relKey; }, - [](SyncItem const& item) -> std::any { return static_cast(&item); }); - }; - - // The fold captures pointers into the Observed's underlying vector; Observed - // stores its value in-place and the pointers stay valid for as long as the - // vector isn't reassigned. setRoots() consumes the nodes synchronously so - // the pointers only need to survive until the tree finishes its keyed merge. - uploadTree_.setRoots(fold(uploadItems_.value())); - downloadTree_.setRoots(fold(downloadItems_.value())); - deleteTree_.setRoots(fold(deleteItems_.value())); -} - -void SyncDialog::Implementation::enqueueSingleByRelKey( - Nui::Observed>& list, std::string const& relKey) -{ - auto const& items = list.value(); - std::size_t targetIdx = items.size(); - for (std::size_t idx = 0; idx < items.size(); ++idx) - { - if (items[idx].relKey == relKey) - { - targetIdx = idx; - break; - } - } - if (targetIdx == items.size()) - return; - - enqueueSingle(list, targetIdx); - - // Backend always transfers the whole subtree for a directory-level - // operation, but only the triggering SyncItem gets a progress observer - // from enqueueSingle. Share it with every descendant SyncItem (by - // relKey prefix) so their rows reflect the same gradient/done/error - // state instead of staying indefinitely "pending". - auto updated = list.value(); - if (targetIdx >= updated.size()) - return; - auto targetProgress = updated[targetIdx].progress; - if (!targetProgress) - return; - const std::string prefix = relKey + "/"; - bool anyChanged = false; - for (auto& item : updated) - { - if (item.relKey.size() > prefix.size() && item.relKey.starts_with(prefix)) - { - item.progress = targetProgress; - anyChanged = true; - } - } - if (anyChanged) - { - list = std::move(updated); - refreshTrees(); - Nui::globalEventContext.executeActiveEventsImmediately(); - } -} - // ---- SyncDialog ------------------------------------------------------------- SyncDialog::SyncDialog(ConfirmDialog* confirmDialog, OperationQueue* operationQueue) @@ -1146,40 +1306,27 @@ SyncDialog::~SyncDialog() SyncDialog::SyncDialog(SyncDialog&&) = default; SyncDialog& SyncDialog::operator=(SyncDialog&&) = default; -void SyncDialog::setOnRecompare( - std::function, - std::vector - )> - )> callback -) +void SyncDialog::setOnRecompareRequested(std::function callback) { - impl_->onRecompare_ = std::move(callback); + impl_->onRecompareRequested_ = std::move(callback); } void SyncDialog::open( + BackendSyncProvider* provider, + SharedData::Sync::DiffSummary summary, std::filesystem::path localPath, - std::filesystem::path remotePath, - std::vector localEntries, - std::vector remoteEntries + std::filesystem::path remotePath ) { + impl_->provider_ = provider; impl_->localPath_ = std::move(localPath); impl_->remotePath_ = std::move(remotePath); - impl_->localEntries_ = std::move(localEntries); - impl_->remoteEntries_ = std::move(remoteEntries); - impl_->recomputeDiff(); + impl_->applySummaryAndReseed(std::move(summary), /*collapseZero=*/false); - impl_->uploadCollapsed_ = impl_->uploadItems_.value().empty(); - impl_->downloadCollapsed_ = impl_->downloadItems_.value().empty(); - impl_->deleteCollapsed_ = impl_->deleteItems_.value().empty(); + impl_->uploadCollapsed_ = impl_->summary_.value().uploads.itemCount == 0; + impl_->downloadCollapsed_ = impl_->summary_.value().downloads.itemCount == 0; + impl_->deleteCollapsed_ = impl_->summary_.value().deletes.itemCount == 0; impl_->open_ = true; impl_->minimized_ = false; @@ -1199,9 +1346,6 @@ Nui::ElementRenderer SyncDialog::operator()() using Nui::Elements::label; namespace Snc = ScriptNuiComponents; - // Localized labels for the direction select. Strings are also used as - // the comparison values when mapping back to the SyncDirection enum; a - // mid-dialog language switch will desync directionStr_ until reopen. const std::vector directionOptions{ language->get("syncDialog", "directionBoth"), language->get("syncDialog", "directionUploadOnly"), @@ -1210,17 +1354,11 @@ Nui::ElementRenderer SyncDialog::operator()() const std::string directionUploadOnly = directionOptions[1]; const std::string directionDownloadOnly = directionOptions[2]; - auto onSettingChange = [this]() - { + auto onSettingChange = [this]() { impl_->recomputeDiff(); Nui::globalEventContext.executeActiveEventsImmediately(); }; - // Row content is now rendered inside the tree — see - // `Implementation::makeTreeRowRenderer`. The 4-cell grid layout lives in - // `.sync-diff-row` and is unchanged; the tree wraps it with indent + - // chevron outside. - // clang-format off return div{ class_ = "sync-dialog-blocker", @@ -1231,8 +1369,6 @@ Nui::ElementRenderer SyncDialog::operator()() }), onClick = [this](Nui::val event) { event.call("stopPropagation"); - // Backdrop click minimizes (preserves state) instead of closing. - // The explicit X button in the dialog header closes/resets. impl_->minimized_ = true; if (impl_->operationQueue_) { @@ -1252,9 +1388,7 @@ Nui::ElementRenderer SyncDialog::operator()() event.call("stopPropagation"); } }( - // ---------------------------------------------------------------- // Header - // ---------------------------------------------------------------- div{class_ = "sync-dialog-header"}( iconPanel({ .icon = Ui5Icons::synchronize(), @@ -1308,11 +1442,8 @@ Nui::ElementRenderer SyncDialog::operator()() }) ), - // ---------------------------------------------------------------- // Body - // ---------------------------------------------------------------- div{class_ = "sync-dialog-body"}( - // Settings panel — groups rendered as cards div{class_ = "sync-settings-panel"}( div{class_ = "sync-settings-card"}( label{class_ = "sync-settings-label"}(language->getObserved("syncDialog", "directionLabel")), @@ -1409,7 +1540,7 @@ Nui::ElementRenderer SyncDialog::operator()() // Diff body — three collapsible sections div{class_ = "sync-diff-body"}( - // ---- Upload section ---- + // Upload div{class_ = "sync-diff-section"}( div{ class_ = "sync-diff-section-header", @@ -1428,14 +1559,14 @@ Nui::ElementRenderer SyncDialog::operator()() ), Ui5Icons::upload(), span{class_ = "sync-section-label"}( - observe(impl_->uploadItems_), + observe(impl_->summary_), [this]() -> Nui::ElementRenderer { using Nui::Elements::span; - const auto totals = computeTotals(impl_->uploadItems_.value()); + auto const& s = impl_->summary_.value().uploads; return span{}(fmt::format( fmt::runtime(language->get("syncDialog", "uploadSectionCount")), - totals.count, - Utility::formatBytes(static_cast(totals.bytes)))); + s.itemCount, + Utility::formatBytes(static_cast(s.transferBytes)))); } ) ), @@ -1449,7 +1580,7 @@ Nui::ElementRenderer SyncDialog::operator()() ) ), - // ---- Download section ---- + // Download div{class_ = "sync-diff-section"}( div{ class_ = "sync-diff-section-header", @@ -1468,14 +1599,14 @@ Nui::ElementRenderer SyncDialog::operator()() ), Ui5Icons::download(), span{class_ = "sync-section-label"}( - observe(impl_->downloadItems_), + observe(impl_->summary_), [this]() -> Nui::ElementRenderer { using Nui::Elements::span; - const auto totals = computeTotals(impl_->downloadItems_.value()); + auto const& s = impl_->summary_.value().downloads; return span{}(fmt::format( fmt::runtime(language->get("syncDialog", "downloadSectionCount")), - totals.count, - Utility::formatBytes(static_cast(totals.bytes)))); + s.itemCount, + Utility::formatBytes(static_cast(s.transferBytes)))); } ) ), @@ -1489,7 +1620,7 @@ Nui::ElementRenderer SyncDialog::operator()() ) ), - // ---- Delete section ---- + // Delete div{class_ = "sync-diff-section"}( div{ class_ = "sync-diff-section-header", @@ -1508,30 +1639,24 @@ Nui::ElementRenderer SyncDialog::operator()() ), Ui5Icons::delete_(), span{class_ = "sync-section-label"}( - observe(impl_->deleteItems_), + observe(impl_->summary_), [this]() -> Nui::ElementRenderer { using Nui::Elements::span; - const auto totals = computeTotals(impl_->deleteItems_.value()); + auto const& s = impl_->summary_.value().deletes; return span{}(fmt::format( fmt::runtime(language->get("syncDialog", "deleteSectionCount")), - totals.count, - Utility::formatBytes(static_cast(totals.bytes)))); + s.itemCount, + Utility::formatBytes(static_cast(s.transferBytes)))); } ) ), div{ - // The Delete tree is one-sided and switches sides - // with the current direction (DeleteRemote → right, - // DeleteLocal → left). Tree::Options::mirror is - // static, so we layer the mirror class on this - // wrapper instead — CSS uses descendant selectors - // (`.script-nui-tree--mirrored .script-nui-tree__row` - // etc.) so the same styling kicks in either way. - class_ = observe(impl_->deleteItems_).generate([this]() -> std::string { - auto const& items = impl_->deleteItems_.value(); - const bool mirror = !items.empty() - && items.front().action == SyncItemAction::DeleteRemote; - return mirror + // Delete rows mirror when the direction favors the + // remote side (DeleteRemote). We infer from the + // current direction setting — with direction=Upload + // deletes happen on remote. + class_ = observe(impl_->directionStr_).generate([this]() -> std::string { + return impl_->direction_ == SyncDirection::Upload ? std::string{"sync-diff-section-rows script-nui-tree--mirrored"} : std::string{"sync-diff-section-rows"}; }), @@ -1545,104 +1670,99 @@ Nui::ElementRenderer SyncDialog::operator()() ) ), - // ---------------------------------------------------------------- // Footer - // ---------------------------------------------------------------- div{class_ = "sync-dialog-footer"}( div{class_ = "sync-footer-actions"}( - Snc::button({ - .text = language->getObserved("syncDialog", "recompare"), - .icon = Ui5Icons::refresh(), - .attributes = { - onClick = [this](Nui::val) { - if (impl_->onRecompare_) - { - impl_->onRecompare_( - impl_->localPath_, - impl_->remotePath_, - impl_->respectIgnore_.value(), - impl_->recursive_.value(), - impl_->ignoreHidden_.value(), - [this](auto localE, auto remoteE) { - open( - impl_->localPath_, - impl_->remotePath_, - std::move(localE), - std::move(remoteE) - ); - } - ); + Snc::button({ + .text = language->getObserved("syncDialog", "recompare"), + .icon = Ui5Icons::refresh(), + .attributes = { + onClick = [this](Nui::val) { + if (impl_->onRecompareRequested_) + impl_->onRecompareRequested_(RecompareRequest{ + .respectIgnoreFiles = impl_->respectIgnore_.value(), + .recursive = impl_->recursive_.value(), + .ignoreHidden = impl_->ignoreHidden_.value(), + .diffOptions = impl_->buildDiffOptions(), + }); } } - } - }), - div{class_ = "sync-queue-status"}( - observe(impl_->operationQueue_->pausedState()), - [this]() -> Nui::ElementRenderer { - using Nui::Elements::div; - using Nui::Elements::span; - if (impl_->operationQueue_->pausedState().value()) - { - return Snc::button({ - .text = language->getObserved("syncDialog", "resumeQueue"), - .icon = Ui5Icons::play(), - .attributes = { - onClick = [this](Nui::val) { impl_->operationQueue_->unpause(); } - }, - .styleVariant = Snc::StyleVariant::Success, - }); + }), + div{class_ = "sync-queue-status"}( + observe(impl_->operationQueue_->pausedState()), + [this]() -> Nui::ElementRenderer { + using Nui::Elements::div; + using Nui::Elements::span; + if (impl_->operationQueue_->pausedState().value()) + { + return Snc::button({ + .text = language->getObserved("syncDialog", "resumeQueue"), + .icon = Ui5Icons::play(), + .attributes = { + onClick = [this](Nui::val) { impl_->operationQueue_->unpause(); } + }, + .styleVariant = Snc::StyleVariant::Success, + }); + } + return div{class_ = "sync-queue-running-indicator"}( + div{class_ = "sync-queue-running-dot"}(), + span{}(language->getObserved("syncDialog", "queueRunning")) + ); } - return div{class_ = "sync-queue-running-indicator"}( - div{class_ = "sync-queue-running-dot"}(), - span{}(language->getObserved("syncDialog", "queueRunning")) - ); - } - ), - div{class_ = "sync-footer-summary"}( - observe( - *impl_->uploadSelected_, *impl_->downloadSelected_, *impl_->deleteSelected_, - impl_->uploadItems_, impl_->downloadItems_, impl_->deleteItems_ ), - [this]() -> Nui::ElementRenderer { - using Nui::Elements::span; - const auto up = computeTotals(impl_->uploadItems_.value(), &impl_->uploadSelected_->value()); - const auto down = computeTotals(impl_->downloadItems_.value(), &impl_->downloadSelected_->value()); - const auto del = computeTotals(impl_->deleteItems_.value(), &impl_->deleteSelected_->value()); - const std::size_t total = up.count + down.count + del.count; - const std::uint64_t bytes = up.bytes + down.bytes + del.bytes; - return span{}(fmt::format( - fmt::runtime(language->get("syncDialog", "footerSummary")), - total, - Utility::formatBytes(static_cast(bytes)) - )); - } - ), - Snc::button({ - .text = language->getObserved("syncDialog", "synchronize"), - .icon = Ui5Icons::synchronize(), - .attributes = { - onClick = [this](Nui::val) { - impl_->confirmDialog_->open({ - .styleVariant = Snc::StyleVariant::Warning, - .headerText = language->get("syncDialog", "confirmHeader"), - .text = fmt::format( - fmt::runtime(language->get("syncDialog", "confirmText")), - impl_->localPath_.filename().string(), - impl_->remotePath_.filename().string() - ), - .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, - .onClose = [this](std::optional btn) { - if (btn == ConfirmDialog::Button::Yes) - impl_->enqueueOperations(); - } - }); + div{class_ = "sync-footer-summary"}( + observe( + *impl_->uploadSelected_, *impl_->downloadSelected_, *impl_->deleteSelected_, + impl_->summary_ + ), + [this]() -> Nui::ElementRenderer { + using Nui::Elements::span; + // We don't know per-leaf sizes without walking the loaded rows, so + // the footer shows the selected-count across sections and the + // summed section transferBytes as an approximation when the + // user selects all. For partial selections this undercounts + // only when a one-sided bulk-directory is partially deselected + // — an edge case the backend planner handles correctly. + auto const& s = impl_->summary_.value(); + const auto selectedCount = + impl_->uploadSelected_->value().size() + + impl_->downloadSelected_->value().size() + + impl_->deleteSelected_->value().size(); + const auto totalBytes = + s.uploads.transferBytes + s.downloads.transferBytes + s.deletes.transferBytes; + return span{}(fmt::format( + fmt::runtime(language->get("syncDialog", "footerSummary")), + selectedCount, + Utility::formatBytes(static_cast(totalBytes)) + )); } - }, - .styleVariant = Snc::StyleVariant::Warning - }) - ) // sync-footer-actions - ) // sync-dialog-footer - ) // sync-dialog - ); // sync-dialog-blocker + ), + Snc::button({ + .text = language->getObserved("syncDialog", "synchronize"), + .icon = Ui5Icons::synchronize(), + .attributes = { + onClick = [this](Nui::val) { + impl_->confirmDialog_->open({ + .styleVariant = Snc::StyleVariant::Warning, + .headerText = language->get("syncDialog", "confirmHeader"), + .text = fmt::format( + fmt::runtime(language->get("syncDialog", "confirmText")), + impl_->localPath_.filename().string(), + impl_->remotePath_.filename().string() + ), + .buttons = ConfirmDialog::Button::Yes | ConfirmDialog::Button::No, + .onClose = [this](std::optional btn) { + if (btn == ConfirmDialog::Button::Yes) + impl_->enqueueOperations(); + } + }); + } + }, + .styleVariant = Snc::StyleVariant::Warning + }) + ) + ) + ) + ); // clang-format on } diff --git a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp index cd79f20a..3d195388 100644 --- a/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp +++ b/frontend/source/frontend/sync_dialog/sync_progress_dialog.cpp @@ -7,7 +7,8 @@ #include #include -#include +#include +#include #include #include @@ -29,9 +30,7 @@ #include #include #include -#include #include -#include struct SyncProgressDialog::Implementation { @@ -56,32 +55,14 @@ struct SyncProgressDialog::Implementation std::filesystem::path localPath_{}; std::filesystem::path remotePath_{}; - std::function localEntries, - std::vector remoteEntries - )> - onDone_{}; + BackendSyncProvider* provider_{nullptr}; + SharedData::Sync::DiffOptions initialOptions_{}; + std::function onDone_{}; - // Intermediate storage while waiting for both scans to complete - std::optional> localEntries_{}; - std::optional> remoteEntries_{}; - - // Cancelled flag; replaced on each open() to invalidate stale captures + // Cancel token; replaced on each open() to invalidate stale callback captures. + // The provider also has its own cancel flag that kills the backend walk — this + // one only guards the frontend lambdas. std::shared_ptr cancelToken_{std::make_shared(false)}; - - void checkBothComplete(SyncProgressDialog* /*dlg*/) - { - if (!localEntries_ || !remoteEntries_) - return; - - // TODO: add Comparing phase here when hash/diff step is implemented - phase_ = SharedData::SyncPhase::Done; - open_ = false; - Nui::globalEventContext.executeActiveEventsImmediately(); - - if (onDone_) - onDone_(std::move(*localEntries_), std::move(*remoteEntries_)); - } }; SyncProgressDialog::SyncProgressDialog(OperationQueue* operationQueue) @@ -94,24 +75,27 @@ SyncProgressDialog::SyncProgressDialog(SyncProgressDialog&&) = default; SyncProgressDialog& SyncProgressDialog::operator=(SyncProgressDialog&&) = default; void SyncProgressDialog::open( + BackendSyncProvider* provider, std::filesystem::path localPath, std::filesystem::path remotePath, bool respectIgnoreFiles, bool recursive, bool ignoreHidden, - std::function localEntries, - std::vector remoteEntries - )> onDone + SharedData::Sync::DiffOptions initialOptions, + std::function onDone ) { - // Cancel any in-progress scan + // Invalidate any stale callbacks from a previous open() by flipping the old + // token and swapping in a fresh one. (The new backend provider has its own + // cancel flag that will also bail the current walk if one is running.) *impl_->cancelToken_ = true; auto token = std::make_shared(false); impl_->cancelToken_ = token; + impl_->provider_ = provider; impl_->localPath_ = std::move(localPath); impl_->remotePath_ = std::move(remotePath); + impl_->initialOptions_ = initialOptions; impl_->onDone_ = std::move(onDone); // Reset state @@ -120,48 +104,59 @@ void SyncProgressDialog::open( impl_->remoteListed_ = 0ull; impl_->compared_ = 0ull; impl_->hashProgressBar_.max(0); - impl_->localEntries_.reset(); - impl_->remoteEntries_.reset(); impl_->open_ = true; Nui::globalEventContext.executeActiveEventsImmediately(); - operationQueue_->enqueueSyncScans( + provider->open( impl_->localPath_, impl_->remotePath_, respectIgnoreFiles, recursive, ignoreHidden, - // onRemoteProgress + // onLocalListing [this, token](SharedData::ScanProgress const& progress) { if (*token) return; - impl_->remoteListed_ = progress.totalScanned; + impl_->localListed_ = progress.totalScanned; Nui::globalEventContext.executeActiveEventsImmediately(); }, - // onLocalProgress + // onRemoteListing [this, token](SharedData::ScanProgress const& progress) { if (*token) return; - impl_->localListed_ = progress.totalScanned; + impl_->remoteListed_ = progress.totalScanned; Nui::globalEventContext.executeActiveEventsImmediately(); }, - // onRemoteComplete - [this, token](SharedData::SyncScanResult result) + // onBothListed + [this, token]() { if (*token) return; - impl_->remoteEntries_ = std::move(result.entries); - impl_->checkBothComplete(this); + impl_->phase_ = SharedData::SyncPhase::Comparing; + Nui::globalEventContext.executeActiveEventsImmediately(); + impl_->provider_->recompute( + impl_->initialOptions_, + [this, token](SharedData::Sync::DiffSummary summary) + { + if (*token) + return; + impl_->phase_ = SharedData::SyncPhase::Done; + impl_->open_ = false; + Nui::globalEventContext.executeActiveEventsImmediately(); + if (impl_->onDone_) + impl_->onDone_(std::move(summary)); + } + ); }, - // onLocalComplete - [this, token](SharedData::SyncScanResult result) + // onDiffProgress + [this, token](std::uint64_t compared) { if (*token) return; - impl_->localEntries_ = std::move(result.entries); - impl_->checkBothComplete(this); + impl_->compared_ = compared; + Nui::globalEventContext.executeActiveEventsImmediately(); } ); } @@ -172,8 +167,8 @@ void SyncProgressDialog::cancel() impl_->cancelToken_ = std::make_shared(false); impl_->open_ = false; impl_->phase_ = SharedData::SyncPhase::Idle; - impl_->localEntries_.reset(); - impl_->remoteEntries_.reset(); + if (impl_->provider_) + impl_->provider_->cancelDiff(); Nui::globalEventContext.executeActiveEventsImmediately(); } diff --git a/frontend/source/frontend/terminal/file_engine.cpp b/frontend/source/frontend/terminal/file_engine.cpp index 096c291e..92a1733c 100644 --- a/frontend/source/frontend/terminal/file_engine.cpp +++ b/frontend/source/frontend/terminal/file_engine.cpp @@ -186,9 +186,10 @@ void FileEngine::createFile( ); } -void FileEngine::addSyncScans( +void FileEngine::openSyncSession( std::filesystem::path const& localPath, std::filesystem::path const& remotePath, + Ids::SyncSessionId syncSessionId, Ids::OperationId remoteScanId, Ids::OperationId localScanId, bool respectIgnoreFiles, @@ -198,34 +199,35 @@ void FileEngine::addSyncScans( ) { Log::info( - "Requesting to add sync scan: local='{}' remote='{}'", + "Requesting openSyncSession: local='{}' remote='{}'", localPath.generic_string(), remotePath.generic_string() ); lazyOpen( - [this, localPath, remotePath, remoteScanId, localScanId, respectIgnoreFiles, recursive, ignoreHidden, - onComplete = std::move(onComplete)](auto const& channelId, std::string const& info) + [this, localPath, remotePath, syncSessionId, remoteScanId, localScanId, respectIgnoreFiles, recursive, + ignoreHidden, onComplete = std::move(onComplete)](auto const& channelId, std::string const& info) { if (!channelId) { - Log::error("Cannot add sync scan, no channel"); + Log::error("Cannot open sync session, no channel"); onComplete(false, info); return; } Nui::RpcClient::callWithBackChannel( - fmt::format("Session::{}::sftp::addSyncScan", impl_->engine->sshSessionId().value()), + fmt::format("Session::{}::sftp::openSyncSession", impl_->engine->sshSessionId().value()), [onComplete = std::move(onComplete)](Nui::val val) { if (val.hasOwnProperty("error")) { - Log::error("(Frontend) Failed to add sync scan: {}", val["error"].as()); + Log::error("(Frontend) Failed to open sync session: {}", val["error"].as()); onComplete(false, val["error"].as()); return; } onComplete(true, "Success"); }, channelId.value().value(), + syncSessionId.value(), remoteScanId.value(), localScanId.value(), remotePath.generic_string(), @@ -238,6 +240,138 @@ void FileEngine::addSyncScans( ); } +void FileEngine::recomputeSyncDiff( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffOptions options, + std::function onSummary +) +{ + Nui::RpcClient::callWithBackChannel( + fmt::format("Session::{}::sftp::recomputeSyncDiff", impl_->engine->sshSessionId().value()), + [onSummary = std::move(onSummary)](Nui::val val) + { + if (val.hasOwnProperty("error")) + { + Log::error("recomputeSyncDiff failed: {}", val["error"].as()); + SharedData::Sync::DiffSummary empty{}; + empty.cancelled = true; + onSummary(std::move(empty)); + return; + } + try + { + const auto json = nlohmann::json::parse(Nui::JSON::stringify(val)); + onSummary(json.get()); + } + catch (std::exception const& exc) + { + Log::error("Failed to parse DiffSummary: {}", exc.what()); + SharedData::Sync::DiffSummary empty{}; + empty.cancelled = true; + onSummary(std::move(empty)); + } + }, + syncSessionId.value(), + options + ); +} + +void FileEngine::loadSyncDiffChildren( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::string const& parentRelKey, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected +) +{ + Nui::RpcClient::callWithBackChannel( + fmt::format("Session::{}::sftp::loadSyncDiffChildren", impl_->engine->sshSessionId().value()), + [onResolved = std::move(onResolved), onRejected = std::move(onRejected)](Nui::val val) + { + if (val.hasOwnProperty("error")) + { + onRejected(val["error"].as()); + return; + } + try + { + const auto json = nlohmann::json::parse(Nui::JSON::stringify(val)); + std::vector nodes; + nodes.reserve(json.size()); + for (auto const& elem : json) + nodes.push_back(elem.get()); + onResolved(std::move(nodes)); + } + catch (std::exception const& exc) + { + onRejected(exc.what()); + } + }, + syncSessionId.value(), + section, + parentRelKey, + std::to_string(generation) + ); +} + +void FileEngine::buildSyncEnqueuePlan( + Ids::SyncSessionId syncSessionId, + SharedData::Sync::DiffSection section, + std::vector selectedRelKeys, + std::uint64_t generation, + std::function)> onResolved, + std::function onRejected +) +{ + Nui::RpcClient::callWithBackChannel( + fmt::format("Session::{}::sftp::buildSyncEnqueuePlan", impl_->engine->sshSessionId().value()), + [onResolved = std::move(onResolved), onRejected = std::move(onRejected)](Nui::val val) + { + if (val.hasOwnProperty("error")) + { + onRejected(val["error"].as()); + return; + } + try + { + const auto json = nlohmann::json::parse(Nui::JSON::stringify(val)); + std::vector plan; + plan.reserve(json.size()); + for (auto const& elem : json) + plan.push_back(elem.get()); + onResolved(std::move(plan)); + } + catch (std::exception const& exc) + { + onRejected(exc.what()); + } + }, + syncSessionId.value(), + section, + selectedRelKeys, + std::to_string(generation) + ); +} + +void FileEngine::cancelSyncDiff(Ids::SyncSessionId syncSessionId) +{ + Nui::RpcClient::callWithBackChannel( + fmt::format("Session::{}::sftp::cancelSyncDiff", impl_->engine->sshSessionId().value()), + [](Nui::val /*val*/) {}, + syncSessionId.value() + ); +} + +void FileEngine::closeSyncSession(Ids::SyncSessionId syncSessionId) +{ + Nui::RpcClient::callWithBackChannel( + fmt::format("Session::{}::sftp::closeSyncSession", impl_->engine->sshSessionId().value()), + [](Nui::val /*val*/) {}, + syncSessionId.value() + ); +} + void FileEngine::addDownload( NuiFileExplorer::Item const& remotePath, NuiFileExplorer::Item const& localPath, diff --git a/ids/include/ids/ids.hpp b/ids/include/ids/ids.hpp index fc73ef3d..5cc56b48 100644 --- a/ids/include/ids/ids.hpp +++ b/ids/include/ids/ids.hpp @@ -8,4 +8,5 @@ DEFINE_ID_TYPE(ChannelId) DEFINE_ID_TYPE(UiSessionId) DEFINE_ID_TYPE(FileId) DEFINE_ID_TYPE(OperationId) -DEFINE_ID_TYPE(InstanceId) \ No newline at end of file +DEFINE_ID_TYPE(InstanceId) +DEFINE_ID_TYPE(SyncSessionId) \ No newline at end of file diff --git a/shared_data/include/shared_data/file_operations/sync_scan_result.hpp b/shared_data/include/shared_data/file_operations/sync_scan_result.hpp deleted file mode 100644 index 77b7f67c..00000000 --- a/shared_data/include/shared_data/file_operations/sync_scan_result.hpp +++ /dev/null @@ -1,42 +0,0 @@ -#pragma once - -#include -#include -#include - -#include - -#include -#include -#include - -namespace SharedData -{ - struct SyncScanResult - { - Ids::OperationId operationId{}; - bool isLocal{false}; - std::uint64_t totalBytes{0}; - std::vector entries{}; - }; - - BOOST_DESCRIBE_STRUCT(SyncScanResult, (), (operationId, isLocal, totalBytes, entries)) - - inline void to_json(nlohmann::json& j, SyncScanResult const& res) - { - j = nlohmann::json{ - {"operationId", res.operationId}, - {"isLocal", res.isLocal}, - {"totalBytes", res.totalBytes}, - {"entries", res.entries}, - }; - } - - inline void from_json(nlohmann::json const& j, SyncScanResult& res) - { - j.at("operationId").get_to(res.operationId); - j.at("isLocal").get_to(res.isLocal); - j.at("totalBytes").get_to(res.totalBytes); - j.at("entries").get_to(res.entries); - } -} diff --git a/shared_data/include/shared_data/sync/diff.hpp b/shared_data/include/shared_data/sync/diff.hpp index fdf8112e..6c1ae605 100644 --- a/shared_data/include/shared_data/sync/diff.hpp +++ b/shared_data/include/shared_data/sync/diff.hpp @@ -1,29 +1,21 @@ #pragma once -#include +#include +#include + +#include +#include #include -#include -#include #include -#include namespace SharedData::Sync { - enum class Direction : std::uint8_t - { - Both, - Upload, - Download - }; + struct DiffTreeNode; - enum class Action : std::uint8_t - { - Upload, - Download, - DeleteLocal, - DeleteRemote - }; + BOOST_DEFINE_ENUM_CLASS(Direction, Both, Upload, Download) + + BOOST_DEFINE_ENUM_CLASS(Action, Upload, Download, DeleteLocal, DeleteRemote) /** * @brief Inputs that govern which differences become actionable items. @@ -39,44 +31,126 @@ namespace SharedData::Sync /// When true, entries with any path segment starting with '.' are dropped. bool ignoreHidden{false}; }; + BOOST_DESCRIBE_STRUCT( + DiffOptions, + (), + (direction, actionUpload, actionDownload, actionDelete, recursive, ignoreHidden) + ) /** - * @brief One actionable difference between the local and remote scan results. + * @brief Sink that receives diff-tree emissions during @ref diffScanTrees. * - * The relKey is a posix-style path relative to the corresponding scan root and - * doubles as a stable identifier across recompares. + * The implementation owns the destination storage and pagination layout. + * Progress heartbeats are fired every @p checkpointInterval compares; + * implementations can also ask the walk to exit early via @ref cancelled(). */ - struct DiffEntry + class DiffSink { - std::string relKey; - Action action; - std::optional local; - std::optional remote; - }; + public: + virtual ~DiffSink() = default; - struct DiffResult - { - std::vector uploads; - std::vector downloads; - std::vector deletes; + virtual void emitUpload(DiffTreeNode node, std::string const& parentRelKey) = 0; + virtual void emitDownload(DiffTreeNode node, std::string const& parentRelKey) = 0; + virtual void emitDelete(DiffTreeNode node, std::string const& parentRelKey) = 0; + + /** + * @brief Called periodically with the running compare count. Implementations + * typically translate this into a remote-progress RPC. + */ + virtual void onProgress(std::uint64_t entriesCompared) = 0; + + /** + * @brief Lets the walk bail out early. Default never-cancels implementation + * is provided so tests don't need to implement it. + */ + virtual bool cancelled() const + { + return false; + } }; /** - * @brief Returns true when two entries should be treated as differing. + * @brief True when two scan nodes should count as a difference. * - * Symlinks compare by raw link target (size/mtime of the link itself are - * meaningless). Type mismatches always count as a difference. + * Type mismatches always count. Symlinks compare by raw @ref ScanNode::linkTarget + * (size/mtime of the link metadata itself is meaningless). Files compare by + * (size, mtime). Directories alone never "differ" — differences among their + * children are the diff's job, handled by the merge walk. */ - bool entriesDiffer(DirectoryEntry const& localEntry, DirectoryEntry const& remoteEntry); + bool entriesDiffer(ScanNode const& localNode, ScanNode const& remoteNode); /** - * @brief Computes the upload / download / delete diff lists from two scan results. + * @brief Parallel merge walk of two sorted scan trees. + * + * Assumes both @p local and @p remote have children sorted ascending by name + * (as produced by @ref TreeDirectoryWalker). Emits to @p sink, respecting + * @p options: + * + * - one-sided subtrees emit exactly one node (no recursion), + * - `!options.recursive` suppresses both recursion and one-sided deletes + * whose @ref ScanNode::childrenKnown is false, + * - `options.ignoreHidden` filters per-name at every level. + * + * @see DiffSink::onProgress heartbeats fire every 512 compares. */ - DiffResult computeSyncDiff( - std::filesystem::path const& localRoot, - std::filesystem::path const& remoteRoot, - std::vector const& localEntries, - std::vector const& remoteEntries, - DiffOptions const& options + void diffScanTrees( + ScanNode const& local, + ScanNode const& remote, + DiffOptions const& options, + DiffSink& sink ); + + inline void to_json(nlohmann::json& j, DiffOptions const& s) + { + SharedData::to_json(j, s); + } + inline void from_json(nlohmann::json const& j, DiffOptions& s) + { + SharedData::from_json(j, s); + } + + // Enum ADL bridges — see the matching comment in diff_tree_node.hpp for + // why these are needed even though SharedData::to_json exists. + inline void to_json(nlohmann::json& j, Direction const& e) + { + SharedData::to_json(j, e); + } + inline void from_json(nlohmann::json const& j, Direction& e) + { + SharedData::from_json(j, e); + } + inline void to_json(nlohmann::json& j, Action const& e) + { + SharedData::to_json(j, e); + } + inline void from_json(nlohmann::json const& j, Action& e) + { + SharedData::from_json(j, e); + } + +#ifdef NUI_FRONTEND + // Mirror the to_json/from_json ADL bridges for the val-based serializer. + // Sync-namespace enums aren't found by ADL through SharedData::to_val, so + // we forward here. + inline void to_val(Nui::val& v, Direction const& e) + { + SharedData::to_val(v, e); + } + inline void from_val(Nui::val const& v, Direction& e) + { + SharedData::from_val(v, e); + } + inline void to_val(Nui::val& v, Action const& e) + { + SharedData::to_val(v, e); + } + inline void from_val(Nui::val const& v, Action& e) + { + SharedData::from_val(v, e); + } + // Intentionally no to_val/from_val forwarder for DiffOptions — the + // described-struct overload in nui's convertToVal picks it up automatically, + // and defining ADL hooks here would make HasToVal ambiguous + // against the generic path. +#endif } diff --git a/shared_data/include/shared_data/sync/diff_summary.hpp b/shared_data/include/shared_data/sync/diff_summary.hpp new file mode 100644 index 00000000..b77dd5ff --- /dev/null +++ b/shared_data/include/shared_data/sync/diff_summary.hpp @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include + +#include + +namespace SharedData::Sync +{ + /** + * @brief Aggregate metrics for one section (uploads, downloads, or deletes) of a diff. + */ + struct SectionSummary + { + /// Total number of actionable entries in the whole section (leaves + directories). + std::uint64_t itemCount{0}; + /// Sum of transfer bytes across the section. Directories contribute their + /// subtree total; files contribute their own size. + std::uint64_t transferBytes{0}; + /// Number of top-level rows the frontend should seed into the tree. + std::uint64_t rootChildCount{0}; + }; + BOOST_DESCRIBE_STRUCT(SectionSummary, (), (itemCount, transferBytes, rootChildCount)) + + /** + * @brief Returned synchronously from recomputeSyncDiff; replaces the old + * entries-per-section vectors that used to cross RPC. + */ + struct DiffSummary + { + Ids::SyncSessionId sessionId{}; + SectionSummary uploads{}; + SectionSummary downloads{}; + SectionSummary deletes{}; + /// Count of scan-node pairs/singletons visited during the merge walk. + std::uint64_t entriesCompared{0}; + /// Monotonic token. Frontend passes this back on subsequent loadChildren calls + /// so stale requests issued before a new recompute can be rejected. + std::uint64_t generation{0}; + /// Backend-side decision: input was large enough that Comparing-phase progress + /// events were emitted. Frontend uses this for telemetry only; rendering the + /// Comparing phase is unconditional. + bool heavyCompare{false}; + /// True when the walk bailed out early because of a cancel request. + bool cancelled{false}; + }; + BOOST_DESCRIBE_STRUCT( + DiffSummary, + (), + (sessionId, uploads, downloads, deletes, entriesCompared, generation, heavyCompare, cancelled) + ) + + // ADL hooks — nlohmann's adl_serializer looks for to_json/from_json in the type's + // namespace, and SharedData::to_json lives one namespace up. These forwarders + // let it find the generic described-struct path. + inline void to_json(nlohmann::json& j, SectionSummary const& s) + { + SharedData::to_json(j, s); + } + inline void from_json(nlohmann::json const& j, SectionSummary& s) + { + SharedData::from_json(j, s); + } + inline void to_json(nlohmann::json& j, DiffSummary const& s) + { + SharedData::to_json(j, s); + } + inline void from_json(nlohmann::json const& j, DiffSummary& s) + { + SharedData::from_json(j, s); + } + +#ifdef NUI_FRONTEND + // No to_val/from_val forwarders for SectionSummary / DiffSummary — the + // described-struct overload in nui's convertToVal handles them natively. +#endif +} diff --git a/shared_data/include/shared_data/sync/diff_tree_node.hpp b/shared_data/include/shared_data/sync/diff_tree_node.hpp new file mode 100644 index 00000000..c46581f2 --- /dev/null +++ b/shared_data/include/shared_data/sync/diff_tree_node.hpp @@ -0,0 +1,105 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +namespace SharedData::Sync +{ + BOOST_DEFINE_ENUM_CLASS(DiffSection, Upload, Download, Delete) + + /** + * @brief One row of the diff tree, as sent to the frontend. + * + * Carries just enough to render the row and drive selection — not the full + * DirectoryEntry. One-sided directories emit a single node whose + * @ref directChildCount / @ref descendantItemCount describe the subtree + * without forcing the walk to recurse. + * + * The frontend treats @ref relKey as the stable node id (doubles as the + * selection key). + */ + struct DiffTreeNode + { + /// Stable identifier. Posix-relative path from the scan root. + std::string relKey{}; + /// Final path segment — what the row renders. + std::string name{}; + Action action{Action::Upload}; + bool isDirectory{false}; + bool hasLocalSide{false}; + bool hasRemoteSide{false}; + std::uint64_t localSize{0}; + std::uint64_t remoteSize{0}; + std::uint64_t localMtime{0}; + std::uint64_t remoteMtime{0}; + /// 0 = leaf or fully-resolved empty dir; >0 = lazy-loadable via + /// loadSyncDiffChildren. + std::uint32_t directChildCount{0}; + /// For one-sided directory emissions: number of actionable descendants + /// (files). Lets the row show a subtree summary without round-tripping. + std::uint64_t descendantItemCount{0}; + std::uint64_t descendantByteTotal{0}; + /// Synthesized ancestor row whose sole purpose is to let the frontend + /// build the tree hierarchy above differing leaves (a/b/c.txt → needs + /// rows for 'a' and 'a/b'). Has no action; its 'action' field is + /// meaningless. Both hasLocalSide and hasRemoteSide are true. + bool isStructural{false}; + }; + BOOST_DESCRIBE_STRUCT( + DiffTreeNode, + (), + (relKey, + name, + action, + isDirectory, + hasLocalSide, + hasRemoteSide, + localSize, + remoteSize, + localMtime, + remoteMtime, + directChildCount, + descendantItemCount, + descendantByteTotal, + isStructural) + ) + + inline void to_json(nlohmann::json& j, DiffTreeNode const& s) + { + SharedData::to_json(j, s); + } + inline void from_json(nlohmann::json const& j, DiffTreeNode& s) + { + SharedData::from_json(j, s); + } + + // Enum-as-string ADL bridges — nlohmann's adl_serializer for DiffSection + // looks for free functions in its own namespace, and SharedData::to_json + // (generic) isn't found by ADL from nested namespaces. + inline void to_json(nlohmann::json& j, DiffSection const& e) + { + SharedData::to_json(j, e); + } + inline void from_json(nlohmann::json const& j, DiffSection& e) + { + SharedData::from_json(j, e); + } + +#ifdef NUI_FRONTEND + inline void to_val(Nui::val& v, DiffSection const& e) + { + SharedData::to_val(v, e); + } + inline void from_val(Nui::val const& v, DiffSection& e) + { + SharedData::from_val(v, e); + } + // No to_val/from_val forwarder for DiffTreeNode — described-struct path + // in nui's convertToVal handles it natively. +#endif +} diff --git a/shared_data/include/shared_data/sync/enqueue_minimizer.hpp b/shared_data/include/shared_data/sync/enqueue_minimizer.hpp index ecc4c42e..660d1bc6 100644 --- a/shared_data/include/shared_data/sync/enqueue_minimizer.hpp +++ b/shared_data/include/shared_data/sync/enqueue_minimizer.hpp @@ -24,14 +24,29 @@ namespace SharedData::Sync }; /** - * @brief Computes the minimal set of indices to enqueue from @p items. + * @brief Computes the minimal set of indices to enqueue from @p items given a + * SPARSE selection set. * - * Bulk-directory items collapse fully-selected subtrees into a single emission. - * Partially-selected subtrees fall through to per-leaf emission. Non-leaf items - * that are not bulk-directories are never emitted on their own (their leaves are). + * Semantics of the sparse set @p sparseSet: + * - An entry X means "X and every descendant of X is selected" (ancestor + * implication). + * - Callers normally keep the set in its sparsest form by collapsing + * fully-selected sibling groups into the shared parent and filling out + * when a deeper descendant is unchecked. The minimizer tolerates + * non-minimal input (redundant ancestors + descendants) — it just walks + * ancestor-in-set during processing. + * + * Emission rules: + * - Bulk-dir (one-sided directory or delete-dir) effectively selected → + * emit once and cover the whole subtree. + * - Plain file leaf effectively selected → emit. + * - Structural two-sided directory effectively selected → no own emission; + * descendants iterate next and inherit the selection. + * + * "Effectively selected" = in the set OR any ancestor is in the set. */ std::vector minimizeEnqueueIndices( std::vector const& items, - std::unordered_set const& selectedLeafRelKeys + std::unordered_set const& sparseSet ); } diff --git a/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp new file mode 100644 index 00000000..21388df6 --- /dev/null +++ b/shared_data/include/shared_data/sync/enqueue_plan_entry.hpp @@ -0,0 +1,59 @@ +#pragma once + +#include +#include + +#include + +#include +#include + +namespace SharedData::Sync +{ + /** + * @brief One row of the backend-computed enqueue plan. + * + * The backend collapses the frontend's selected relKey set into a minimal list + * that: + * + * - emits a one-sided directory as a single bulk-dir entry when every descendant + * is selected (saves N transfers for N descendants), + * - emits an individual leaf otherwise, + * - skips intermediate directory rows with partial selection — the enqueue + * side creates missing directories as needed via the bulk upload/download + * flow. + * + * Both @ref localAbsPath and @ref remoteAbsPath are always populated (the sides + * the action does not touch simply carry the path that would apply if it did). + */ + struct EnqueuePlanEntry + { + std::string relKey{}; + Action action{Action::Upload}; + std::string localAbsPath{}; + std::string remoteAbsPath{}; + std::uint64_t sizeBytes{0}; + std::uint64_t mtime{0}; + std::uint32_t mtimeNsec{0}; + bool isDirectory{false}; + }; + BOOST_DESCRIBE_STRUCT( + EnqueuePlanEntry, + (), + (relKey, action, localAbsPath, remoteAbsPath, sizeBytes, mtime, mtimeNsec, isDirectory) + ) + + inline void to_json(nlohmann::json& j, EnqueuePlanEntry const& e) + { + SharedData::to_json(j, e); + } + inline void from_json(nlohmann::json const& j, EnqueuePlanEntry& e) + { + SharedData::from_json(j, e); + } + +#ifdef NUI_FRONTEND + // No to_val/from_val forwarder for EnqueuePlanEntry — described-struct path + // in nui's convertToVal handles it natively. +#endif +} diff --git a/shared_data/include/shared_data/sync/scan_node.hpp b/shared_data/include/shared_data/sync/scan_node.hpp new file mode 100644 index 00000000..56547ccf --- /dev/null +++ b/shared_data/include/shared_data/sync/scan_node.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include + +#include +#include +#include +#include +#include + +namespace SharedData::Sync +{ + /** + * @brief One node of a scan tree. + * + * Built on the backend by @ref Utility::TreeDirectoryWalker; never serialized to the + * frontend. Children are sorted ascending by @ref name so a parallel merge walk can + * diff against another sorted ScanNode without intermediate maps. + */ + struct ScanNode + { + /// Final path segment. Empty for the root. + std::string name{}; + FileType type{FileType::Unknown}; + std::uint64_t size{0}; + std::uint64_t mtime{0}; + std::uint32_t mtimeNsec{0}; + std::filesystem::perms permissions{std::filesystem::perms::unknown}; + /// For symlinks: the raw link literal. What the diff compares by. + std::optional linkTarget{}; + std::uint32_t uid{0}; + std::uint32_t gid{0}; + /// Sorted ascending by name (POSIX byte comparison). + std::vector children{}; + /// Cached subtree metrics — populated once at build time, reused by the + /// diff to emit "upload-only subtree" summaries without re-walking. + std::uint64_t subtreeFileCount{0}; + std::uint64_t subtreeByteTotal{0}; + /// False when the scan did not descend into this directory (non-recursive mode). + /// Diff suppresses one-sided delete emissions when this is false. + bool childrenKnown{true}; + }; +} diff --git a/shared_data/include/shared_data/sync/tree_directory_walker.hpp b/shared_data/include/shared_data/sync/tree_directory_walker.hpp new file mode 100644 index 00000000..ef358621 --- /dev/null +++ b/shared_data/include/shared_data/sync/tree_directory_walker.hpp @@ -0,0 +1,207 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace SharedData::Sync +{ + /** + * @brief Directory walker that produces a sorted @ref ScanNode tree directly. + * + * Mirrors the scanner contract of @ref Utility::DeepDirectoryWalker — the caller + * supplies a scanner that returns a batch of @ref DirectoryEntry for one directory — + * but internally stitches results into a nested ScanNode tree with: + * + * - sibling sort by name applied per directory as entries arrive, and + * - @ref ScanNode::subtreeFileCount / @ref ScanNode::subtreeByteTotal accumulated + * when the subtree completes. + * + * No second pass is needed. Pointer stability relies on reserving each node's children + * vector before pushing — a node's children vector is only appended to in a single + * scanner invocation, so a `reserve(N) + N pushes` sequence never reallocates. + */ + template + requires std::is_invocable_r_v< + std::expected, WalkErrorType>, + ScannerT, + std::filesystem::path const&> + class TreeDirectoryWalker : public Utility::BaseDirectoryWalker + { + public: + template + requires std::is_same_v, ScannerT> + TreeDirectoryWalker(std::filesystem::path rootPath, ForwardingScannerT&& scanner) + : rootPath_{std::move(rootPath)} + , scanner_{std::forward(scanner)} + { + root_.type = SharedData::FileType::Directory; + root_.childrenKnown = true; + queue_.push_back(QueuedDir{.node = &root_, .fullPath = rootPath_, .depth = 0}); + ++totalEntries_; + } + + /** + * @brief Moves the built tree out of the walker. Must only be called after + * @ref completed() returns true. + */ + ScanNode ejectTree() && + { + finalizeCounts(root_); + return std::move(root_); + } + + std::size_t totalEntries() const override + { + return totalEntries_; + } + + bool completed() const override + { + return queue_.empty(); + } + + void reset() + { + root_ = ScanNode{}; + root_.type = SharedData::FileType::Directory; + root_.childrenKnown = true; + queue_.clear(); + queue_.push_back(QueuedDir{.node = &root_, .fullPath = rootPath_, .depth = 0}); + currentIndex_ = 0; + totalBytes_ = 0; + totalEntries_ = 1; + } + + /** + * @brief Scan one directory from the BFS queue. @return true when the walker is done. + */ + std::expected walk() + { + if (queue_.empty()) + return true; + + const auto pending = queue_.front(); + queue_.pop_front(); + + auto result = scanner_(pending.fullPath); + if (!result) + return std::unexpected(std::move(result).error()); + + auto rawChildren = std::move(result).value(); + + if constexpr (ScannerIncludesDotAndDotDot) + { + std::erase_if(rawChildren, [](SharedData::DirectoryEntry const& entry) { + return entry.path == "." || entry.path == ".."; + }); + } + + std::sort( + rawChildren.begin(), + rawChildren.end(), + [](SharedData::DirectoryEntry const& lhs, SharedData::DirectoryEntry const& rhs) { + return lhs.path.filename().generic_string() < rhs.path.filename().generic_string(); + } + ); + + // Reserve up front so `&children.back()` stays valid through every push_back + // inside this loop — lets us enqueue directory pointers in the same pass. + pending.node->children.reserve(rawChildren.size()); + for (auto& raw : rawChildren) + { + if (raw.type == SharedData::FileType::Regular) + totalBytes_ += raw.size; + + pending.node->children.push_back(ScanNode{ + .name = raw.path.filename().generic_string(), + .type = raw.type, + .size = raw.size, + .mtime = raw.mtime, + .mtimeNsec = raw.mtimeNsec, + .permissions = raw.permissions, + .linkTarget = std::move(raw.linkTarget), + .uid = raw.uid, + .gid = raw.gid, + .children = {}, + .subtreeFileCount = 0, + .subtreeByteTotal = 0, + .childrenKnown = true, + }); + ++totalEntries_; + + if (raw.type == SharedData::FileType::Directory) + { + auto& inserted = pending.node->children.back(); + queue_.push_back(QueuedDir{ + .node = &inserted, + .fullPath = pending.fullPath / inserted.name, + .depth = pending.depth + 1, + }); + } + } + + ++currentIndex_; + return queue_.empty(); + } + + std::expected walkAll() + { + decltype(walk()) res; + do + { + res = walk(); + if (!res) + return std::unexpected(std::move(res).error()); + } while (!res.value()); + return {}; + } + + private: + /** + * @brief Post-order subtree-metric fill. Runs once, on eject. + */ + static void finalizeCounts(ScanNode& node) + { + node.subtreeFileCount = 0; + node.subtreeByteTotal = 0; + for (auto& child : node.children) + { + if (child.type == SharedData::FileType::Directory) + { + finalizeCounts(child); + node.subtreeFileCount += child.subtreeFileCount; + node.subtreeByteTotal += child.subtreeByteTotal; + } + else + { + node.subtreeFileCount += 1; + node.subtreeByteTotal += child.size; + } + } + } + + struct QueuedDir + { + ScanNode* node; + std::filesystem::path fullPath; + std::size_t depth; + }; + + std::filesystem::path rootPath_; + ScannerT scanner_; + ScanNode root_{}; + std::deque queue_{}; + std::size_t totalEntries_{0}; + }; +} diff --git a/shared_data/source/shared_data/sync/diff.cpp b/shared_data/source/shared_data/sync/diff.cpp index 5060f153..600c0108 100644 --- a/shared_data/source/shared_data/sync/diff.cpp +++ b/shared_data/source/shared_data/sync/diff.cpp @@ -1,205 +1,335 @@ #include +#include -#include +#include +#include +#include #include +#include namespace SharedData::Sync { namespace { - std::map - buildEntryMap(std::filesystem::path const& root, std::vector const& entries) + constexpr std::uint64_t progressCheckpointInterval = 512; + + bool isHidden(std::string const& name) { - std::map result; - for (std::size_t idx = 1; idx < entries.size(); ++idx) - { - auto const& entry = entries[idx]; - std::filesystem::path relPath; - if (entry.fullPath.has_relative_path()) - { - relPath = std::filesystem::path{entry.fullPath.generic_string()}.lexically_relative( - std::filesystem::path{root.generic_string()} - ); - } - else - { - relPath = entry.path; - } - if (!relPath.empty()) - result.emplace(relPath.generic_string(), entry); - } - return result; + return !name.empty() && name.front() == '.'; } - bool hasHiddenSegment(std::string const& relKey) + std::string joinRel(std::string const& parentRelKey, std::string const& name) { - std::size_t pos = 0; - while (pos < relKey.size()) + if (parentRelKey.empty()) + return name; + std::string out; + out.reserve(parentRelKey.size() + 1 + name.size()); + out.append(parentRelKey); + out.push_back('/'); + out.append(name); + return out; + } + + /** + * @brief Resolve how a one-sided local entry should be emitted. + * Returns std::nullopt when the entry should be skipped. + */ + std::optional chooseLocalOnlyAction(ScanNode const& node, DiffOptions const& options) + { + if (options.direction == Direction::Download) { - if (relKey[pos] == '.') - return true; - pos = relKey.find('/', pos); - if (pos == std::string::npos) - break; - ++pos; + if (!options.actionDelete) + return std::nullopt; + // Non-recursive mode: if we never descended into this directory we cannot + // safely delete it — the user hasn't seen its contents. + if (node.type == FileType::Directory && !node.childrenKnown) + return std::nullopt; + return Action::DeleteLocal; } - return false; + if (!options.actionUpload) + return std::nullopt; + return Action::Upload; } - bool isNested(std::string const& relKey) + /** + * @brief Mirror of @ref chooseLocalOnlyAction for a one-sided remote entry. + */ + std::optional chooseRemoteOnlyAction(ScanNode const& node, DiffOptions const& options) { - return relKey.find('/') != std::string::npos; + if (options.direction == Direction::Upload) + { + if (!options.actionDelete) + return std::nullopt; + if (node.type == FileType::Directory && !node.childrenKnown) + return std::nullopt; + return Action::DeleteRemote; + } + if (!options.actionDownload) + return std::nullopt; + return Action::Download; } - } - bool entriesDiffer(DirectoryEntry const& localEntry, DirectoryEntry const& remoteEntry) - { - if (localEntry.type != remoteEntry.type) - return true; - if (localEntry.type == FileType::Symlink) + DiffTreeNode makeLocalOnlyRow(ScanNode const& node, std::string const& relKey, Action action) { - if (localEntry.linkTarget && remoteEntry.linkTarget) - return *localEntry.linkTarget != *remoteEntry.linkTarget; - return false; + const bool isDir = node.type == FileType::Directory; + return DiffTreeNode{ + .relKey = relKey, + .name = node.name, + .action = action, + .isDirectory = isDir, + .hasLocalSide = true, + .hasRemoteSide = false, + .localSize = node.size, + .remoteSize = 0, + .localMtime = node.mtime, + .remoteMtime = 0, + .directChildCount = isDir ? static_cast(node.children.size()) : 0u, + .descendantItemCount = isDir ? node.subtreeFileCount : 1ull, + .descendantByteTotal = isDir ? node.subtreeByteTotal : node.size, + }; } - if (localEntry.size != remoteEntry.size) - return true; - if (localEntry.mtime != remoteEntry.mtime) - return true; - return false; - } - - DiffResult computeSyncDiff( - std::filesystem::path const& localRoot, - std::filesystem::path const& remoteRoot, - std::vector const& localEntries, - std::vector const& remoteEntries, - DiffOptions const& options - ) - { - DiffResult result; - if (localEntries.empty() && remoteEntries.empty()) - return result; - - auto localMap = buildEntryMap(localRoot, localEntries); - auto remoteMap = buildEntryMap(remoteRoot, remoteEntries); - - if (options.ignoreHidden) + DiffTreeNode makeRemoteOnlyRow(ScanNode const& node, std::string const& relKey, Action action) { - for (auto mapIter = localMap.begin(); mapIter != localMap.end();) - mapIter = hasHiddenSegment(mapIter->first) ? localMap.erase(mapIter) : std::next(mapIter); - for (auto mapIter = remoteMap.begin(); mapIter != remoteMap.end();) - mapIter = hasHiddenSegment(mapIter->first) ? remoteMap.erase(mapIter) : std::next(mapIter); + const bool isDir = node.type == FileType::Directory; + return DiffTreeNode{ + .relKey = relKey, + .name = node.name, + .action = action, + .isDirectory = isDir, + .hasLocalSide = false, + .hasRemoteSide = true, + .localSize = 0, + .remoteSize = node.size, + .localMtime = 0, + .remoteMtime = node.mtime, + .directChildCount = isDir ? static_cast(node.children.size()) : 0u, + .descendantItemCount = isDir ? node.subtreeFileCount : 1ull, + .descendantByteTotal = isDir ? node.subtreeByteTotal : node.size, + }; } - if (!options.recursive) + DiffTreeNode makeFileDifferRow( + ScanNode const& localNode, + ScanNode const& remoteNode, + std::string const& relKey, + Action action + ) { - for (auto mapIter = localMap.begin(); mapIter != localMap.end();) - mapIter = isNested(mapIter->first) ? localMap.erase(mapIter) : std::next(mapIter); - for (auto mapIter = remoteMap.begin(); mapIter != remoteMap.end();) - mapIter = isNested(mapIter->first) ? remoteMap.erase(mapIter) : std::next(mapIter); + return DiffTreeNode{ + .relKey = relKey, + .name = localNode.name, + .action = action, + .isDirectory = false, + .hasLocalSide = true, + .hasRemoteSide = true, + .localSize = localNode.size, + .remoteSize = remoteNode.size, + .localMtime = localNode.mtime, + .remoteMtime = remoteNode.mtime, + .directChildCount = 0, + .descendantItemCount = 1, + .descendantByteTotal = action == Action::Upload ? localNode.size : remoteNode.size, + }; } - // ---- Entries that exist locally ---------------------------------- - for (auto const& [relKey, localEntry] : localMap) + void emitOneSided( + DiffTreeNode row, + Action action, + std::string const& parentRelKey, + DiffSink& sink + ) { - auto remoteIter = remoteMap.find(relKey); - if (remoteIter == remoteMap.end()) + switch (action) { - if (options.actionUpload && options.direction != Direction::Download) - { - result.uploads.push_back(DiffEntry{ - .relKey = relKey, - .action = Action::Upload, - .local = localEntry, - .remote = std::nullopt, - }); - } - else if (options.actionDelete && options.direction == Direction::Download) - { - // In non-recursive mode the children of this directory weren't scanned, - // so deleting it could remove items the user never saw — hide it. - const bool skipDir = !options.recursive && localEntry.type == FileType::Directory; - if (!skipDir) - { - result.deletes.push_back(DiffEntry{ - .relKey = relKey, - .action = Action::DeleteLocal, - .local = localEntry, - .remote = std::nullopt, - }); - } - } - continue; + case Action::Upload: + sink.emitUpload(std::move(row), parentRelKey); + return; + case Action::Download: + sink.emitDownload(std::move(row), parentRelKey); + return; + case Action::DeleteLocal: + case Action::DeleteRemote: + sink.emitDelete(std::move(row), parentRelKey); + return; } + } - auto const& remoteEntry = remoteIter->second; + /** + * @brief Forward declaration — recursion point for directory pairs. + */ + void mergeChildren( + std::vector const& localChildren, + std::vector const& remoteChildren, + std::string const& parentRelKey, + DiffOptions const& options, + DiffSink& sink, + std::uint64_t& comparedCounter + ); - if (localEntry.type == FileType::Directory) - continue; - if (!entriesDiffer(localEntry, remoteEntry)) - continue; + void handleBothPresent( + ScanNode const& localChild, + ScanNode const& remoteChild, + std::string const& relKey, + std::string const& parentRelKey, + DiffOptions const& options, + DiffSink& sink, + std::uint64_t& comparedCounter + ) + { + const bool sameDir = + localChild.type == FileType::Directory && remoteChild.type == FileType::Directory; + if (sameDir) + { + if (options.recursive) + mergeChildren(localChild.children, remoteChild.children, relKey, options, sink, comparedCounter); + return; + } - const bool localNewer = localEntry.mtime >= remoteEntry.mtime; + if (!entriesDiffer(localChild, remoteChild)) + return; - if (options.direction == Direction::Upload || - (options.direction == Direction::Both && localNewer)) + // Direction routing: Upload-only → Upload; Download-only → Download; + // Both → mtime-newer wins (local >= remote favors Upload). + if (options.direction == Direction::Upload) { if (options.actionUpload) - { - result.uploads.push_back(DiffEntry{ - .relKey = relKey, - .action = Action::Upload, - .local = localEntry, - .remote = remoteEntry, - }); - } + sink.emitUpload(makeFileDifferRow(localChild, remoteChild, relKey, Action::Upload), parentRelKey); + return; + } + if (options.direction == Direction::Download) + { + if (options.actionDownload) + sink.emitDownload( + makeFileDifferRow(localChild, remoteChild, relKey, Action::Download), parentRelKey + ); + return; + } + // Both direction. + const bool localNewer = localChild.mtime >= remoteChild.mtime; + if (localNewer) + { + if (options.actionUpload) + sink.emitUpload(makeFileDifferRow(localChild, remoteChild, relKey, Action::Upload), parentRelKey); } else { if (options.actionDownload) - { - result.downloads.push_back(DiffEntry{ - .relKey = relKey, - .action = Action::Download, - .local = localEntry, - .remote = remoteEntry, - }); - } + sink.emitDownload( + makeFileDifferRow(localChild, remoteChild, relKey, Action::Download), parentRelKey + ); } } - // ---- Entries that exist only remotely ---------------------------- - for (auto const& [relKey, remoteEntry] : remoteMap) + void mergeChildren( + std::vector const& localChildren, + std::vector const& remoteChildren, + std::string const& parentRelKey, + DiffOptions const& options, + DiffSink& sink, + std::uint64_t& comparedCounter + ) { - if (localMap.count(relKey)) - continue; + std::size_t li = 0; + std::size_t ri = 0; - if (options.actionDownload && options.direction != Direction::Upload) - { - result.downloads.push_back(DiffEntry{ - .relKey = relKey, - .action = Action::Download, - .local = std::nullopt, - .remote = remoteEntry, - }); - } - else if (options.actionDelete && options.direction == Direction::Upload) + const auto bumpAndCheckpoint = [&]() { + ++comparedCounter; + if (comparedCounter % progressCheckpointInterval == 0) + sink.onProgress(comparedCounter); + }; + + while (li < localChildren.size() || ri < remoteChildren.size()) { - const bool skipDir = !options.recursive && remoteEntry.type == FileType::Directory; - if (!skipDir) + if (sink.cancelled()) + return; + + const bool hasL = li < localChildren.size(); + const bool hasR = ri < remoteChildren.size(); + + // Pick the smaller name (or the only side available). + const bool takeBoth = hasL && hasR && localChildren[li].name == remoteChildren[ri].name; + const bool takeLeft = hasL && (!hasR || localChildren[li].name < remoteChildren[ri].name); + const bool takeRight = hasR && (!hasL || remoteChildren[ri].name < localChildren[li].name); + + if (takeBoth) + { + auto const& lc = localChildren[li]; + auto const& rc = remoteChildren[ri]; + if (!options.ignoreHidden || !isHidden(lc.name)) + { + const auto relKey = joinRel(parentRelKey, lc.name); + handleBothPresent(lc, rc, relKey, parentRelKey, options, sink, comparedCounter); + } + ++li; + ++ri; + bumpAndCheckpoint(); + } + else if (takeLeft) + { + auto const& lc = localChildren[li]; + if (!options.ignoreHidden || !isHidden(lc.name)) + { + if (auto action = chooseLocalOnlyAction(lc, options); action) + { + const auto relKey = joinRel(parentRelKey, lc.name); + emitOneSided(makeLocalOnlyRow(lc, relKey, *action), *action, parentRelKey, sink); + } + } + ++li; + bumpAndCheckpoint(); + } + else if (takeRight) { - result.deletes.push_back(DiffEntry{ - .relKey = relKey, - .action = Action::DeleteRemote, - .local = std::nullopt, - .remote = remoteEntry, - }); + auto const& rc = remoteChildren[ri]; + if (!options.ignoreHidden || !isHidden(rc.name)) + { + if (auto action = chooseRemoteOnlyAction(rc, options); action) + { + const auto relKey = joinRel(parentRelKey, rc.name); + emitOneSided(makeRemoteOnlyRow(rc, relKey, *action), *action, parentRelKey, sink); + } + } + ++ri; + bumpAndCheckpoint(); + } + else + { + // Both exhausted — loop will exit. + break; } } } + } - return result; + bool entriesDiffer(ScanNode const& localNode, ScanNode const& remoteNode) + { + if (localNode.type != remoteNode.type) + return true; + if (localNode.type == FileType::Symlink) + { + if (localNode.linkTarget && remoteNode.linkTarget) + return *localNode.linkTarget != *remoteNode.linkTarget; + return false; + } + if (localNode.size != remoteNode.size) + return true; + if (localNode.mtime != remoteNode.mtime) + return true; + return false; + } + + void diffScanTrees( + ScanNode const& local, + ScanNode const& remote, + DiffOptions const& options, + DiffSink& sink + ) + { + std::uint64_t compared = 0; + mergeChildren(local.children, remote.children, std::string{}, options, sink, compared); + // Final heartbeat so observers see the completed count even if it didn't + // land on a checkpoint boundary. + sink.onProgress(compared); } } diff --git a/shared_data/source/shared_data/sync/enqueue_minimizer.cpp b/shared_data/source/shared_data/sync/enqueue_minimizer.cpp index 74c30692..2333d6e0 100644 --- a/shared_data/source/shared_data/sync/enqueue_minimizer.cpp +++ b/shared_data/source/shared_data/sync/enqueue_minimizer.cpp @@ -5,79 +5,62 @@ namespace SharedData::Sync { + namespace + { + /** @brief Walks every parent prefix of @p relKey (excluding @p relKey + * itself) and returns true when any of them is present in + * @p sparseSet. "Effective selection" in the sparse model: + * an entry X in the set implies every descendant of X is + * selected too. + */ + bool anyAncestorInSet( + std::string const& relKey, + std::unordered_set const& sparseSet + ) + { + std::string_view view{relKey}; + while (true) + { + const auto slash = view.rfind('/'); + if (slash == std::string_view::npos) + return false; + view.remove_suffix(view.size() - slash); + if (sparseSet.contains(std::string{view})) + return true; + } + } + } + std::vector minimizeEnqueueIndices( std::vector const& items, - std::unordered_set const& selectedLeafRelKeys + std::unordered_set const& sparseSet ) { const std::size_t count = items.size(); if (count == 0) return {}; - std::vector isTreeLeaf(count, true); - for (std::size_t idx = 0; idx < count; ++idx) - { - const std::string prefix = items[idx].relKey + "/"; - for (std::size_t other = 0; other < count; ++other) - { - if (other == idx) - continue; - if (items[other].relKey.size() > prefix.size() && - items[other].relKey.starts_with(prefix)) - { - isTreeLeaf[idx] = false; - break; - } - } - } - - // For a directory-role item the "any" / "all selected" questions reduce to - // scanning its descendant tree-leaves. - const auto subtreeSelectionState = [&](std::size_t idx) { - struct State - { - bool any{false}; - bool all{true}; - } state; - if (isTreeLeaf[idx]) - { - const bool checked = selectedLeafRelKeys.contains(items[idx].relKey); - state.any = checked; - state.all = checked; - return state; - } - const std::string prefix = items[idx].relKey + "/"; - bool sawAnyLeaf = false; - for (std::size_t other = 0; other < count; ++other) - { - if (!isTreeLeaf[other]) - continue; - if (items[other].relKey.size() <= prefix.size() || - !items[other].relKey.starts_with(prefix)) - continue; - sawAnyLeaf = true; - const bool checked = selectedLeafRelKeys.contains(items[other].relKey); - state.any = state.any || checked; - if (!checked) - state.all = false; - } - if (!sawAnyLeaf) - state.all = false; - return state; - }; + // Precompute "tree-leaf" flag — true when no other item is a strict + // descendant. For the sparse minimizer this only matters for pretty + // diagnostic reasons; the emission rules below don't actually use it. + // Kept around in case a future callers want it. + // Process in ascending relKey length so ancestors are handled before + // their descendants. That lets bulk-dir coverage short-circuit the + // descendants, and structural-dir membership flows down naturally via + // anyAncestorInSet. std::vector order(count); std::iota(order.begin(), order.end(), std::size_t{0}); std::sort(order.begin(), order.end(), [&](std::size_t lhs, std::size_t rhs) { return items[lhs].relKey.size() < items[rhs].relKey.size(); }); - std::vector coveredPrefixes; + std::vector bulkCoveredPrefixes; std::vector result; result.reserve(count); - const auto isCovered = [&](std::string const& relKey) { - for (auto const& ancestor : coveredPrefixes) + const auto isBulkCovered = [&](std::string const& relKey) { + for (auto const& ancestor : bulkCoveredPrefixes) { const std::string prefix = ancestor + "/"; if (relKey.size() > prefix.size() && relKey.starts_with(prefix)) @@ -86,32 +69,61 @@ namespace SharedData::Sync return false; }; + // Precompute tree-leaf for disambiguating structural vs file rows: + // a row with isBulkDir=false and any descendant in items is structural; + // with no descendants it's a plain file. + std::vector isTreeLeaf(count, true); + for (std::size_t idx = 0; idx < count; ++idx) + { + const std::string prefix = items[idx].relKey + "/"; + for (std::size_t other = 0; other < count; ++other) + { + if (other == idx) + continue; + if (items[other].relKey.size() > prefix.size() && + items[other].relKey.starts_with(prefix)) + { + isTreeLeaf[idx] = false; + break; + } + } + } + for (std::size_t idx : order) { auto const& item = items[idx]; - if (isCovered(item.relKey)) + + // An ancestor was already emitted as a bulk op — everything below + // it is covered by that single enqueue entry. + if (isBulkCovered(item.relKey)) + continue; + + const bool inSet = sparseSet.contains(item.relKey); + const bool ancestorInSet = anyAncestorInSet(item.relKey, sparseSet); + const bool effectivelySelected = inSet || ancestorInSet; + if (!effectivelySelected) continue; if (item.isBulkDir) { - const auto state = subtreeSelectionState(idx); - if (!state.any) - continue; - if (state.all) - { - result.push_back(idx); - coveredPrefixes.push_back(item.relKey); - continue; - } + // One-sided directory or delete-directory — collapses the + // entire subtree into a single enqueue entry regardless of + // whether the item is in the set itself or just inherits. + result.push_back(idx); + bulkCoveredPrefixes.push_back(item.relKey); continue; } - if (!isTreeLeaf[idx]) - continue; - if (!selectedLeafRelKeys.contains(item.relKey)) + if (isTreeLeaf[idx]) + { + // Plain file leaf (no descendants, not bulk). + result.push_back(idx); continue; + } - result.push_back(idx); + // Structural two-sided directory. No SFTP primitive syncs it as a + // single op, so don't emit the row itself; the descendant leaves + // iterate next and inherit the selection via anyAncestorInSet. } return result; diff --git a/shared_data/test/shared_data/test_diff.hpp b/shared_data/test/shared_data/test_diff.hpp index 002cc1a4..37210fd3 100644 --- a/shared_data/test/shared_data/test_diff.hpp +++ b/shared_data/test/shared_data/test_diff.hpp @@ -1,6 +1,8 @@ #pragma once #include +#include +#include #include @@ -10,330 +12,432 @@ namespace SharedData::Sync::Test { - inline DirectoryEntry - makeEntry(std::string const& relPath, FileType type, std::uint64_t size = 0, std::uint64_t mtime = 0, - std::filesystem::path const& root = {}) + /** + * @brief Test-only sink that captures emissions into flat vectors. Mirrors the + * backend-side DiffTreeStore closely enough to make assertions easy. + */ + struct CollectingSink : DiffSink { - DirectoryEntry entry{}; - entry.path = relPath; - if (!root.empty()) - entry.fullPath = root / relPath; - entry.type = type; - entry.size = size; - entry.mtime = mtime; - return entry; - } + struct Record + { + DiffTreeNode node; + std::string parentRelKey; + }; - inline std::vector - makeScan(std::filesystem::path const& root, std::vector children) - { - std::vector entries; - entries.reserve(children.size() + 1); - // Index 0 is the scan root (excluded from buildEntryMap). - DirectoryEntry rootEntry{}; - rootEntry.path = root; - rootEntry.type = FileType::Directory; - entries.push_back(std::move(rootEntry)); - for (auto& child : children) + std::vector uploads; + std::vector downloads; + std::vector deletes; + std::uint64_t lastProgress{0}; + + void emitUpload(DiffTreeNode node, std::string const& parentRelKey) override { - if (child.fullPath.empty()) - child.fullPath = root / child.path; - entries.push_back(std::move(child)); + uploads.push_back(Record{.node = std::move(node), .parentRelKey = parentRelKey}); } - return entries; + void emitDownload(DiffTreeNode node, std::string const& parentRelKey) override + { + downloads.push_back(Record{.node = std::move(node), .parentRelKey = parentRelKey}); + } + void emitDelete(DiffTreeNode node, std::string const& parentRelKey) override + { + deletes.push_back(Record{.node = std::move(node), .parentRelKey = parentRelKey}); + } + void onProgress(std::uint64_t entriesCompared) override + { + lastProgress = entriesCompared; + } + }; + + inline bool hasRelKey(std::vector const& records, std::string const& relKey) + { + return std::any_of(records.begin(), records.end(), [&](CollectingSink::Record const& record) { + return record.node.relKey == relKey; + }); } - inline bool hasRelKey(std::vector const& entries, std::string const& relKey) + inline CollectingSink::Record const* + findRelKey(std::vector const& records, std::string const& relKey) { - return std::any_of(entries.begin(), entries.end(), - [&](DiffEntry const& entry) { return entry.relKey == relKey; }); + auto iter = std::find_if(records.begin(), records.end(), [&](CollectingSink::Record const& record) { + return record.node.relKey == relKey; + }); + return iter == records.end() ? nullptr : &*iter; } - inline DiffEntry const* findRelKey(std::vector const& entries, std::string const& relKey) + /** + * @brief Builds a leaf file node. + */ + inline ScanNode makeFile(std::string const& name, std::uint64_t size, std::uint64_t mtime) { - auto iter = std::find_if(entries.begin(), entries.end(), - [&](DiffEntry const& entry) { return entry.relKey == relKey; }); - return iter == entries.end() ? nullptr : &*iter; + return ScanNode{ + .name = name, + .type = FileType::Regular, + .size = size, + .mtime = mtime, + }; } - TEST(SyncDiffTests, EmptyInputsProduceNoDiff) + /** + * @brief Builds a symlink node with a given raw link target. + */ + inline ScanNode makeSymlink(std::string const& name, std::filesystem::path const& target, std::uint64_t mtime = 0) { - const auto result = computeSyncDiff("/local", "/remote", {}, {}, {}); - EXPECT_TRUE(result.uploads.empty()); - EXPECT_TRUE(result.downloads.empty()); - EXPECT_TRUE(result.deletes.empty()); + return ScanNode{ + .name = name, + .type = FileType::Symlink, + .size = 0, + .mtime = mtime, + .linkTarget = target, + }; } - TEST(SyncDiffTests, IdenticalTreesProduceNoDiff) + /** + * @brief Builds a directory with the given children, sorts by name, and accumulates + * subtreeFileCount / subtreeByteTotal post-order (mirrors what + * @ref Utility::TreeDirectoryWalker does at scan time). + */ + inline ScanNode makeDir(std::string const& name, std::vector children, bool childrenKnown = true) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; + std::sort(children.begin(), children.end(), [](ScanNode const& lhs, ScanNode const& rhs) { + return lhs.name < rhs.name; + }); - auto local = makeScan(localRoot, {makeEntry("file.txt", FileType::Regular, 17, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {makeEntry("file.txt", FileType::Regular, 17, 100, remoteRoot)}); + ScanNode node{ + .name = name, + .type = FileType::Directory, + .children = std::move(children), + .childrenKnown = childrenKnown, + }; - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - EXPECT_TRUE(result.uploads.empty()); - EXPECT_TRUE(result.downloads.empty()); - EXPECT_TRUE(result.deletes.empty()); + for (auto const& child : node.children) + { + if (child.type == FileType::Directory) + { + node.subtreeFileCount += child.subtreeFileCount; + node.subtreeByteTotal += child.subtreeByteTotal; + } + else + { + node.subtreeFileCount += 1; + node.subtreeByteTotal += child.size; + } + } + return node; } - TEST(SyncDiffTests, LocalOnlyEntryBecomesUploadInBothDirection) + TEST(SyncDiffTests, EmptyTreesProduceNoEmits) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {makeEntry("only_local.txt", FileType::Regular, 5, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {}); - - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - ASSERT_EQ(result.uploads.size(), 1u); - EXPECT_EQ(result.uploads.front().relKey, "only_local.txt"); - EXPECT_EQ(result.uploads.front().action, Action::Upload); - EXPECT_TRUE(result.uploads.front().local.has_value()); - EXPECT_FALSE(result.uploads.front().remote.has_value()); - EXPECT_TRUE(result.downloads.empty()); - EXPECT_TRUE(result.deletes.empty()); + CollectingSink sink; + diffScanTrees(makeDir("", {}), makeDir("", {}), {}, sink); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); } - TEST(SyncDiffTests, RemoteOnlyEntryBecomesDownloadInBothDirection) + TEST(SyncDiffTests, IdenticalTreesProduceNoEmits) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {}); - auto remote = makeScan(remoteRoot, {makeEntry("only_remote.txt", FileType::Regular, 5, 100, remoteRoot)}); - - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - ASSERT_EQ(result.downloads.size(), 1u); - EXPECT_EQ(result.downloads.front().relKey, "only_remote.txt"); - EXPECT_EQ(result.downloads.front().action, Action::Download); + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("file.txt", 17, 100)}), + makeDir("", {makeFile("file.txt", 17, 100)}), + {}, + sink + ); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); } - TEST(SyncDiffTests, NewerLocalEntryWinsInBothDirection) + TEST(SyncDiffTests, LocalOnlyFileBecomesUploadInBothDirection) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {makeEntry("file.txt", FileType::Regular, 5, 200, localRoot)}); - auto remote = makeScan(remoteRoot, {makeEntry("file.txt", FileType::Regular, 5, 100, remoteRoot)}); - // Force a difference in size so the entries are considered to differ; the - // direction is then chosen by mtime. - local[1].size = 6; - - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - EXPECT_EQ(result.uploads.size(), 1u); - EXPECT_TRUE(result.downloads.empty()); + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("only_local.txt", 5, 100)}), + makeDir("", {}), + {}, + sink + ); + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "only_local.txt"); + EXPECT_EQ(sink.uploads.front().node.action, Action::Upload); + EXPECT_TRUE(sink.uploads.front().node.hasLocalSide); + EXPECT_FALSE(sink.uploads.front().node.hasRemoteSide); + EXPECT_EQ(sink.uploads.front().parentRelKey, ""); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); } - TEST(SyncDiffTests, NewerRemoteEntryWinsInBothDirection) + TEST(SyncDiffTests, RemoteOnlyFileBecomesDownloadInBothDirection) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {makeEntry("file.txt", FileType::Regular, 6, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {makeEntry("file.txt", FileType::Regular, 5, 200, remoteRoot)}); - - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - EXPECT_TRUE(result.uploads.empty()); - EXPECT_EQ(result.downloads.size(), 1u); + CollectingSink sink; + diffScanTrees( + makeDir("", {}), + makeDir("", {makeFile("only_remote.txt", 5, 100)}), + {}, + sink + ); + ASSERT_EQ(sink.downloads.size(), 1u); + EXPECT_EQ(sink.downloads.front().node.relKey, "only_remote.txt"); + EXPECT_EQ(sink.downloads.front().node.action, Action::Download); } - TEST(SyncDiffTests, UploadDirectionForcesUploadOnDifferingEntry) + TEST(SyncDiffTests, NewerLocalFileWinsInBothDirection) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("file.txt", 6, 200)}), + makeDir("", {makeFile("file.txt", 5, 100)}), + {}, + sink + ); + EXPECT_EQ(sink.uploads.size(), 1u); + EXPECT_TRUE(sink.downloads.empty()); + } - // Remote has a newer mtime, but Direction::Upload must override. - auto local = makeScan(localRoot, {makeEntry("file.txt", FileType::Regular, 6, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {makeEntry("file.txt", FileType::Regular, 5, 200, remoteRoot)}); + TEST(SyncDiffTests, NewerRemoteFileWinsInBothDirection) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("file.txt", 6, 100)}), + makeDir("", {makeFile("file.txt", 5, 200)}), + {}, + sink + ); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_EQ(sink.downloads.size(), 1u); + } - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, {.direction = Direction::Upload} + TEST(SyncDiffTests, UploadDirectionForcesUploadOnDifferingFile) + { + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("file.txt", 6, 100)}), + makeDir("", {makeFile("file.txt", 5, 200)}), + {.direction = Direction::Upload}, + sink ); - EXPECT_EQ(result.uploads.size(), 1u); - EXPECT_TRUE(result.downloads.empty()); + EXPECT_EQ(sink.uploads.size(), 1u); + EXPECT_TRUE(sink.downloads.empty()); } TEST(SyncDiffTests, DownloadDirectionSuppressesUploads) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {makeEntry("only_local.txt", FileType::Regular, 5, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {}); - - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, {.direction = Direction::Download} + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("only_local.txt", 5, 100)}), + makeDir("", {}), + {.direction = Direction::Download}, + sink ); - EXPECT_TRUE(result.uploads.empty()); - EXPECT_TRUE(result.downloads.empty()); - EXPECT_TRUE(result.deletes.empty()); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); } - TEST(SyncDiffTests, DownloadDirectionWithDeleteRemovesLocalOnlyEntry) + TEST(SyncDiffTests, DownloadDirectionWithDeleteRemovesLocalOnlyFile) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {makeEntry("only_local.txt", FileType::Regular, 5, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {}); - - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, - {.direction = Direction::Download, .actionDelete = true} + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("only_local.txt", 5, 100)}), + makeDir("", {}), + {.direction = Direction::Download, .actionDelete = true}, + sink ); - ASSERT_EQ(result.deletes.size(), 1u); - EXPECT_EQ(result.deletes.front().action, Action::DeleteLocal); + ASSERT_EQ(sink.deletes.size(), 1u); + EXPECT_EQ(sink.deletes.front().node.action, Action::DeleteLocal); } - TEST(SyncDiffTests, UploadDirectionWithDeleteRemovesRemoteOnlyEntry) + TEST(SyncDiffTests, UploadDirectionWithDeleteRemovesRemoteOnlyFile) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {}); - auto remote = makeScan(remoteRoot, {makeEntry("only_remote.txt", FileType::Regular, 5, 100, remoteRoot)}); - - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, - {.direction = Direction::Upload, .actionDelete = true} + CollectingSink sink; + diffScanTrees( + makeDir("", {}), + makeDir("", {makeFile("only_remote.txt", 5, 100)}), + {.direction = Direction::Upload, .actionDelete = true}, + sink ); - ASSERT_EQ(result.deletes.size(), 1u); - EXPECT_EQ(result.deletes.front().action, Action::DeleteRemote); + ASSERT_EQ(sink.deletes.size(), 1u); + EXPECT_EQ(sink.deletes.front().node.action, Action::DeleteRemote); } - TEST(SyncDiffTests, DirectoriesPresentOnBothSidesDoNotProduceDiff) + TEST(SyncDiffTests, SameDirectoryOnBothSidesDoesNotEmitAnything) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {makeEntry("subdir", FileType::Directory, 0, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {makeEntry("subdir", FileType::Directory, 0, 200, remoteRoot)}); - - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - EXPECT_TRUE(result.uploads.empty()); - EXPECT_TRUE(result.downloads.empty()); + CollectingSink sink; + diffScanTrees( + makeDir("", {makeDir("subdir", {})}), + makeDir("", {makeDir("subdir", {})}), + {}, + sink + ); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); } - TEST(SyncDiffTests, NonRecursiveDropsNestedEntries) + TEST(SyncDiffTests, OneSidedDirectoryEmitsSingleRowWithSubtreeCounts) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, { - makeEntry("top.txt", FileType::Regular, 5, 100, localRoot), - makeEntry("subdir/nested.txt", FileType::Regular, 5, 100, localRoot), + // Local has a directory tree with 3 files; remote is empty. A one-sided + // directory should emit exactly one row with directChildCount/descendant counts + // from the cached ScanNode metrics — no recursion during diff. + auto subtree = makeDir("dir", { + makeFile("a.txt", 10, 100), + makeFile("b.txt", 20, 100), + makeDir("inner", {makeFile("c.txt", 5, 100)}), }); - auto remote = makeScan(remoteRoot, {}); - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, {.recursive = false} + CollectingSink sink; + diffScanTrees( + makeDir("", {std::move(subtree)}), + makeDir("", {}), + {}, + sink ); - ASSERT_EQ(result.uploads.size(), 1u); - EXPECT_EQ(result.uploads.front().relKey, "top.txt"); + + ASSERT_EQ(sink.uploads.size(), 1u); + auto const& row = sink.uploads.front().node; + EXPECT_EQ(row.relKey, "dir"); + EXPECT_TRUE(row.isDirectory); + EXPECT_EQ(row.descendantItemCount, 3u); + EXPECT_EQ(row.descendantByteTotal, 10u + 20u + 5u); + EXPECT_EQ(row.directChildCount, 3u); } - TEST(SyncDiffTests, NonRecursiveDeleteHidesDirectoryEntries) + TEST(SyncDiffTests, RecursiveWalkEmitsEachDifferingChild) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - // Local-only directory + Direction::Download + actionDelete; without recursive - // mode the directory is hidden to avoid a recursive delete the user can't audit. - auto local = makeScan(localRoot, {makeEntry("orphan_dir", FileType::Directory, 0, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {}); - - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, - {.direction = Direction::Download, .actionDelete = true, .recursive = false} + CollectingSink sink; + diffScanTrees( + makeDir("", { + makeDir("sub", { + makeFile("a.txt", 10, 100), + makeFile("b.txt", 20, 100), + }), + }), + makeDir("", { + makeDir("sub", {}), + }), + {}, + sink ); - EXPECT_TRUE(result.deletes.empty()); + ASSERT_EQ(sink.uploads.size(), 2u); + EXPECT_TRUE(hasRelKey(sink.uploads, "sub/a.txt")); + EXPECT_TRUE(hasRelKey(sink.uploads, "sub/b.txt")); + EXPECT_EQ(findRelKey(sink.uploads, "sub/a.txt")->parentRelKey, "sub"); } - TEST(SyncDiffTests, IgnoreHiddenDropsDotEntriesAndTrees) + TEST(SyncDiffTests, NonRecursiveSkipsDescendIntoSameNameDirectory) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; + // top.txt differs → upload. subdir exists on both sides but in non-recursive + // mode the walk must NOT descend into it. + CollectingSink sink; + diffScanTrees( + makeDir("", { + makeFile("top.txt", 5, 100), + makeDir("subdir", {makeFile("nested.txt", 5, 100)}), + }), + makeDir("", { + makeDir("subdir", {}), + }), + {.recursive = false}, + sink + ); + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "top.txt"); + } - auto local = makeScan(localRoot, { - makeEntry("normal.txt", FileType::Regular, 5, 100, localRoot), - makeEntry(".hidden", FileType::Regular, 5, 100, localRoot), - makeEntry(".git/config", FileType::Regular, 5, 100, localRoot), - }); - auto remote = makeScan(remoteRoot, {}); + TEST(SyncDiffTests, NonRecursiveDeleteHidesUnknownChildrenDirectory) + { + // Local has a directory that the scanner didn't descend into + // (childrenKnown=false). With Direction::Download + actionDelete, we must + // suppress the delete — deleting a subtree the user can't audit is unsafe. + CollectingSink sink; + diffScanTrees( + makeDir("", {makeDir("orphan_dir", {}, /*childrenKnown=*/false)}), + makeDir("", {}), + {.direction = Direction::Download, .actionDelete = true, .recursive = false}, + sink + ); + EXPECT_TRUE(sink.deletes.empty()); + } - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, {.ignoreHidden = true} + TEST(SyncDiffTests, IgnoreHiddenDropsDotEntriesAtEveryLevel) + { + CollectingSink sink; + diffScanTrees( + makeDir("", { + makeFile("normal.txt", 5, 100), + makeFile(".hidden", 5, 100), + makeDir(".git", {makeFile("config", 5, 100)}), + }), + makeDir("", {}), + {.ignoreHidden = true}, + sink ); - ASSERT_EQ(result.uploads.size(), 1u); - EXPECT_EQ(result.uploads.front().relKey, "normal.txt"); + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "normal.txt"); } TEST(SyncDiffTests, SymlinksWithSameTargetDoNotDiffer) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto localLink = makeEntry("link", FileType::Symlink, 0, 100, localRoot); - localLink.linkTarget = std::filesystem::path{"/some/target"}; - auto remoteLink = makeEntry("link", FileType::Symlink, 99, 200, remoteRoot); - remoteLink.linkTarget = std::filesystem::path{"/some/target"}; - - auto local = makeScan(localRoot, {localLink}); - auto remote = makeScan(remoteRoot, {remoteLink}); - - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - EXPECT_TRUE(result.uploads.empty()); - EXPECT_TRUE(result.downloads.empty()); + CollectingSink sink; + diffScanTrees( + makeDir("", {makeSymlink("link", "/some/target", 100)}), + makeDir("", {makeSymlink("link", "/some/target", 200)}), + {}, + sink + ); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); } TEST(SyncDiffTests, SymlinksWithDifferentTargetsProduceDiff) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto localLink = makeEntry("link", FileType::Symlink, 0, 200, localRoot); - localLink.linkTarget = std::filesystem::path{"/local/target"}; - auto remoteLink = makeEntry("link", FileType::Symlink, 0, 100, remoteRoot); - remoteLink.linkTarget = std::filesystem::path{"/remote/target"}; - - auto local = makeScan(localRoot, {localLink}); - auto remote = makeScan(remoteRoot, {remoteLink}); - - const auto result = computeSyncDiff(localRoot, remoteRoot, local, remote, {}); - EXPECT_EQ(result.uploads.size() + result.downloads.size(), 1u); + CollectingSink sink; + diffScanTrees( + makeDir("", {makeSymlink("link", "/local/target", 200)}), + makeDir("", {makeSymlink("link", "/remote/target", 100)}), + {}, + sink + ); + EXPECT_EQ(sink.uploads.size() + sink.downloads.size(), 1u); } - TEST(SyncDiffTests, TypeMismatchAlwaysCountsAsDifference) + TEST(SyncDiffTests, TypeMismatchCountsAsDifference) { - DirectoryEntry localEntry{}; - localEntry.type = FileType::Regular; - localEntry.size = 5; - localEntry.mtime = 100; - - DirectoryEntry remoteEntry{}; - remoteEntry.type = FileType::Symlink; - remoteEntry.size = 5; - remoteEntry.mtime = 100; - - EXPECT_TRUE(entriesDiffer(localEntry, remoteEntry)); + auto localNode = makeFile("same_name", 5, 100); + auto remoteNode = makeSymlink("same_name", "/elsewhere", 100); + EXPECT_TRUE(entriesDiffer(localNode, remoteNode)); } TEST(SyncDiffTests, ActionTogglesSuppressMatchingLists) { - const std::filesystem::path localRoot = "/local"; - const std::filesystem::path remoteRoot = "/remote"; - - auto local = makeScan(localRoot, {makeEntry("only_local.txt", FileType::Regular, 5, 100, localRoot)}); - auto remote = makeScan(remoteRoot, {makeEntry("only_remote.txt", FileType::Regular, 5, 100, remoteRoot)}); - - const auto result = computeSyncDiff( - localRoot, remoteRoot, local, remote, - {.actionUpload = false, .actionDownload = false, .actionDelete = false} + CollectingSink sink; + diffScanTrees( + makeDir("", {makeFile("only_local.txt", 5, 100)}), + makeDir("", {makeFile("only_remote.txt", 5, 100)}), + {.actionUpload = false, .actionDownload = false, .actionDelete = false}, + sink ); - EXPECT_TRUE(result.uploads.empty()); - EXPECT_TRUE(result.downloads.empty()); - EXPECT_TRUE(result.deletes.empty()); + EXPECT_TRUE(sink.uploads.empty()); + EXPECT_TRUE(sink.downloads.empty()); + EXPECT_TRUE(sink.deletes.empty()); + } + + TEST(SyncDiffTests, OneSidedFullVsEmptyIsCheaperThanEqualWalk) + { + // Sanity check on the algorithm's "prune one-sided subtrees" promise: + // 1 local child with 100 deep descendants vs empty remote should visit + // exactly one top-level compare, not 100. + std::vector deepChildren; + deepChildren.reserve(100); + for (int idx = 0; idx < 100; ++idx) + deepChildren.push_back(makeFile("f" + std::to_string(idx), 1, 100)); + auto localTree = makeDir("", {makeDir("deep", std::move(deepChildren))}); + + CollectingSink sink; + diffScanTrees(localTree, makeDir("", {}), {}, sink); + + ASSERT_EQ(sink.uploads.size(), 1u); + EXPECT_EQ(sink.uploads.front().node.relKey, "deep"); + // Only the one top-level compare happened. + EXPECT_EQ(sink.lastProgress, 1u); } } diff --git a/shared_data/test/shared_data/test_enqueue_minimizer.hpp b/shared_data/test/shared_data/test_enqueue_minimizer.hpp index f5bc61f7..e8f207a5 100644 --- a/shared_data/test/shared_data/test_enqueue_minimizer.hpp +++ b/shared_data/test/shared_data/test_enqueue_minimizer.hpp @@ -44,23 +44,26 @@ namespace SharedData::Sync::Test EXPECT_TRUE(result.empty()); } - TEST(EnqueueMinimizerTests, FullySelectedSubtreeCollapsesToBulkDir) + TEST(EnqueueMinimizerTests, BulkDirInSetEmitsBulk) { - // dir/ is a bulk dir; both descendants are selected → emit just the dir. + // Sparse model: caller put the bulk dir itself in the set → emit bulk, + // descendants covered. std::vector items{ {.relKey = "dir", .isBulkDir = true}, {.relKey = "dir/a.txt", .isBulkDir = false}, {.relKey = "dir/b.txt", .isBulkDir = false}, }; - std::unordered_set selected{"dir/a.txt", "dir/b.txt"}; + std::unordered_set selected{"dir"}; const auto result = minimizeEnqueueIndices(items, selected); ASSERT_EQ(result.size(), 1u); EXPECT_EQ(result.front(), 0u) << "must emit the bulk dir, not its descendants"; } - TEST(EnqueueMinimizerTests, PartiallySelectedSubtreeEmitsOnlySelectedLeaves) + TEST(EnqueueMinimizerTests, IndividualLeafSelectionEmitsLeavesEvenUnderBulkDir) { + // Sparse model: caller did NOT collapse to the bulk dir; they selected + // one file under it. Minimizer just emits what's in the set. std::vector items{ {.relKey = "dir", .isBulkDir = true}, {.relKey = "dir/a.txt", .isBulkDir = false}, @@ -87,27 +90,25 @@ namespace SharedData::Sync::Test EXPECT_TRUE(result.empty()); } - TEST(EnqueueMinimizerTests, NestedFullySelectedSubtreeIsCoveredByOuterDir) + TEST(EnqueueMinimizerTests, OuterBulkDirCoversNestedBulkAndLeaves) { - // outer/ contains inner/ which contains file. With everything selected, - // only the outer bulk dir should be emitted (covers nested dir + leaf). + // outer is in the sparse set → bulk-emit and cover every descendant. std::vector items{ {.relKey = "outer", .isBulkDir = true}, {.relKey = "outer/inner", .isBulkDir = true}, {.relKey = "outer/inner/leaf.txt", .isBulkDir = false}, }; - std::unordered_set selected{"outer/inner/leaf.txt"}; + std::unordered_set selected{"outer"}; const auto result = minimizeEnqueueIndices(items, selected); ASSERT_EQ(result.size(), 1u); - EXPECT_EQ(result.front(), 0u) << "expected outer dir to cover nested subtree"; + EXPECT_EQ(result.front(), 0u) << "outer bulk emission must cover the whole subtree"; } TEST(EnqueueMinimizerTests, NonBulkIntermediateNodeIsNotEmittedItself) { - // "dir" is NOT a bulk dir (e.g., it represents a file action that happens to share - // a relKey with descendants — the production scenario is rare but the algorithm - // still must not emit it on its own). + // Two-sided structural directory — no SFTP primitive syncs it as one op. + // Even when explicitly in the set it must not emit; its descendants do. std::vector items{ {.relKey = "dir", .isBulkDir = false}, {.relKey = "dir/leaf.txt", .isBulkDir = false}, @@ -119,6 +120,46 @@ namespace SharedData::Sync::Test EXPECT_EQ(result.front(), 1u) << "intermediate non-bulk node must not be emitted"; } + TEST(EnqueueMinimizerTests, StructuralDirInSetExpandsToLeafDescendants) + { + // Sparse set contains a structural (two-sided) dir. No bulk primitive + // applies; every leaf descendant must emit individually via ancestor + // implication. + std::vector items{ + {.relKey = "parent", .isBulkDir = false}, + {.relKey = "parent/a.txt", .isBulkDir = false}, + {.relKey = "parent/sub", .isBulkDir = false}, + {.relKey = "parent/sub/b.txt", .isBulkDir = false}, + }; + std::unordered_set selected{"parent"}; + + const auto result = minimizeEnqueueIndices(items, selected); + EXPECT_FALSE(resultContains(result, 0u)) << "structural dir itself not emitted"; + EXPECT_TRUE(resultContains(result, 1u)); + EXPECT_FALSE(resultContains(result, 2u)) << "nested structural dir not emitted"; + EXPECT_TRUE(resultContains(result, 3u)); + } + + TEST(EnqueueMinimizerTests, StructuralDirInSetStillCollapsesNestedBulk) + { + // Structural outer dir in the sparse set; contains a nested bulk dir. + // The bulk dir should still collapse into a single emission (not + // descend into its own children). + std::vector items{ + {.relKey = "outer", .isBulkDir = false}, + {.relKey = "outer/bulk", .isBulkDir = true}, + {.relKey = "outer/bulk/x.txt", .isBulkDir = false}, + {.relKey = "outer/leaf.txt", .isBulkDir = false}, + }; + std::unordered_set selected{"outer"}; + + const auto result = minimizeEnqueueIndices(items, selected); + EXPECT_FALSE(resultContains(result, 0u)); + EXPECT_TRUE(resultContains(result, 1u)) << "inner bulk dir emitted once"; + EXPECT_FALSE(resultContains(result, 2u)) << "leaf under inner bulk is covered"; + EXPECT_TRUE(resultContains(result, 3u)); + } + TEST(EnqueueMinimizerTests, MixedSelectionAcrossSiblingSubtrees) { std::vector items{ @@ -128,23 +169,23 @@ namespace SharedData::Sync::Test {.relKey = "right/b.txt", .isBulkDir = false}, {.relKey = "right/c.txt", .isBulkDir = false}, }; - // left is fully selected → collapse to bulk dir. - // right is partially selected → emit b.txt only. - std::unordered_set selected{"left/a.txt", "right/b.txt"}; + // Sparse: "left" means "take all of left"; individual leaves under + // "right" are selected atomically. + std::unordered_set selected{"left", "right/b.txt"}; const auto result = minimizeEnqueueIndices(items, selected); EXPECT_TRUE(resultContains(result, 0u)) << "left bulk dir"; - EXPECT_FALSE(resultContains(result, 1u)); - EXPECT_FALSE(resultContains(result, 2u)) << "right not fully selected → no bulk"; + EXPECT_FALSE(resultContains(result, 1u)) << "left/a.txt covered by bulk"; + EXPECT_FALSE(resultContains(result, 2u)) << "right not in set and no leaves cover it"; EXPECT_TRUE(resultContains(result, 3u)); EXPECT_FALSE(resultContains(result, 4u)); } TEST(EnqueueMinimizerTests, BulkDirWithoutDescendantsIsEmittedWhenSelected) { - // A directory item with no descendants in the diff list (e.g., an empty dir - // that needs to be created on the other side) is itself a tree leaf — selecting - // it must cause it to be emitted so the dir actually gets created. + // A directory item with no descendants in the diff list (empty dir + // that needs to be created on the other side). Selecting it must + // cause the row to emit so the dir actually gets created. std::vector items{ {.relKey = "empty_dir", .isBulkDir = true}, }; @@ -154,27 +195,39 @@ namespace SharedData::Sync::Test ASSERT_EQ(result.size(), 1u); EXPECT_EQ(result.front(), 0u); - // Without selection it is suppressed. EXPECT_TRUE(minimizeEnqueueIndices(items, {}).empty()); } - TEST(EnqueueMinimizerTests, OuterDirNotEmittedWhenInnerHasUnselectedLeaf) + TEST(EnqueueMinimizerTests, RedundantAncestorAndDescendantAreTolerated) { + // Non-minimal sparse input: the ancestor already implies the descendant. + // Minimizer should still produce the correct emission (bulk once). std::vector items{ {.relKey = "outer", .isBulkDir = true}, - {.relKey = "outer/a.txt", .isBulkDir = false}, - {.relKey = "outer/inner", .isBulkDir = true}, - {.relKey = "outer/inner/leaf.txt", .isBulkDir = false}, - {.relKey = "outer/inner/other.txt", .isBulkDir = false}, + {.relKey = "outer/inner.txt", .isBulkDir = false}, }; - // Skip outer/inner/other.txt → outer cannot be collapsed. - std::unordered_set selected{"outer/a.txt", "outer/inner/leaf.txt"}; + std::unordered_set selected{"outer", "outer/inner.txt"}; const auto result = minimizeEnqueueIndices(items, selected); - EXPECT_FALSE(resultContains(result, 0u)) << "outer must not be emitted"; - EXPECT_FALSE(resultContains(result, 2u)) << "inner not fully selected either"; + ASSERT_EQ(result.size(), 1u); + EXPECT_EQ(result.front(), 0u); + } + + TEST(EnqueueMinimizerTests, PartialStructuralSubtreeEmitsOnlySelectedLeaves) + { + std::vector items{ + {.relKey = "parent", .isBulkDir = false}, + {.relKey = "parent/a.txt", .isBulkDir = false}, + {.relKey = "parent/b.txt", .isBulkDir = false}, + {.relKey = "parent/c.txt", .isBulkDir = false}, + }; + // Sparse leaf-level selection (user filled out after unchecking one). + std::unordered_set selected{"parent/a.txt", "parent/c.txt"}; + + const auto result = minimizeEnqueueIndices(items, selected); + EXPECT_FALSE(resultContains(result, 0u)); EXPECT_TRUE(resultContains(result, 1u)); + EXPECT_FALSE(resultContains(result, 2u)); EXPECT_TRUE(resultContains(result, 3u)); - EXPECT_FALSE(resultContains(result, 4u)); } }