From fbb8401af8678246dc389dfd399eed8b4d633115 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Sun, 14 Sep 2025 20:11:27 -0500 Subject: [PATCH 1/5] Perform multiple upserts in a single transaction. --- Sources/SQLiteData/CloudKit/SyncEngine.swift | 147 ++++++++++--------- 1 file changed, 77 insertions(+), 70 deletions(-) diff --git a/Sources/SQLiteData/CloudKit/SyncEngine.swift b/Sources/SQLiteData/CloudKit/SyncEngine.swift index 71245e50..76951e90 100644 --- a/Sources/SQLiteData/CloudKit/SyncEngine.swift +++ b/Sources/SQLiteData/CloudKit/SyncEngine.swift @@ -1259,20 +1259,24 @@ case share(CKShare) case reference(CKShare.Reference) } - var shares: [ShareOrReference] = [] - for record in modifications { - if let share = record as? CKShare { - shares.append(.share(share)) - } else { - await upsertFromServerRecord(record) - if let shareReference = record.share { - shares.append(.reference(shareReference)) + let shares = LockIsolated<[ShareOrReference]>([]) + await withErrorReporting(.sqliteDataCloudKitFailure) { + try await userDatabase.write { db in + for record in modifications { + if let share = record as? CKShare { + shares.withValue { $0.append(.share(share)) } + } else { + upsertFromServerRecord(record, db: db) + if let shareReference = record.share { + shares.withValue { $0.append(.reference(shareReference)) } + } + } } } } await withTaskGroup(of: Void.self) { group in - for share in shares { + for share in shares.withValue(\.self) { group.addTask { switch share { case .share(let share): @@ -1495,93 +1499,96 @@ force: Bool = false ) async { await withErrorReporting(.sqliteDataCloudKitFailure) { + try await userDatabase.write { db in + upsertFromServerRecord(serverRecord, force: force, db: db) + } + } + } + + private func upsertFromServerRecord( + _ serverRecord: CKRecord, + force: Bool = false, + db: Database + ) { + withErrorReporting(.sqliteDataCloudKitFailure) { guard let table = tablesByName[serverRecord.recordType] else { guard let recordPrimaryKey = serverRecord.recordID.recordPrimaryKey else { return } - try await userDatabase.write { db in - try SyncMetadata.insert { - SyncMetadata( - recordPrimaryKey: recordPrimaryKey, - recordType: serverRecord.recordType, - parentRecordPrimaryKey: serverRecord.parent?.recordID.recordPrimaryKey, - parentRecordType: serverRecord.parent?.recordID.tableName, - lastKnownServerRecord: serverRecord, - _lastKnownServerRecordAllFields: serverRecord, - share: nil, - userModificationDate: serverRecord.userModificationDate - ) - } onConflict: { - ($0.recordPrimaryKey, $0.recordType) - } doUpdate: { - $0.setLastKnownServerRecord(serverRecord) - } - .execute(db) + try SyncMetadata.insert { + SyncMetadata( + recordPrimaryKey: recordPrimaryKey, + recordType: serverRecord.recordType, + parentRecordPrimaryKey: serverRecord.parent?.recordID.recordPrimaryKey, + parentRecordType: serverRecord.parent?.recordID.tableName, + lastKnownServerRecord: serverRecord, + _lastKnownServerRecordAllFields: serverRecord, + share: nil, + userModificationDate: serverRecord.userModificationDate + ) + } onConflict: { + ($0.recordPrimaryKey, $0.recordType) + } doUpdate: { + $0.setLastKnownServerRecord(serverRecord) } + .execute(db) return } - let metadata = try await metadatabase.read { db in - try SyncMetadata - .where { $0.recordName.eq(serverRecord.recordID.recordName) } - .fetchOne(db) - } + let metadata = try SyncMetadata + .where { $0.recordName.eq(serverRecord.recordID.recordName) } + .fetchOne(db) serverRecord.userModificationDate = - metadata?.userModificationDate ?? serverRecord.userModificationDate + metadata?.userModificationDate ?? serverRecord.userModificationDate - func open(_: T.Type) async throws { + func open(_: T.Type) throws { let columnNames: [String] if !force, let metadata, let allFields = metadata._lastKnownServerRecordAllFields { - columnNames = try await userDatabase.read { db in - var columnNames = T.TableColumns.writableColumns.map(\.name) - let row = try T.find(#sql("\(bind: metadata.recordPrimaryKey)")).fetchOne(db) - guard let row - else { - reportIssue( - """ - Local database record could not be found for '\(serverRecord.recordID.recordName)'. - """ - ) - return columnNames - } + var _columnNames = T.TableColumns.writableColumns.map(\.name) + let row = try T.find(#sql("\(bind: metadata.recordPrimaryKey)")).fetchOne(db) + if let row { serverRecord.update( with: allFields, row: T(queryOutput: row), - columnNames: &columnNames, + columnNames: &_columnNames, parentForeignKey: foreignKeysByTableName[T.tableName]?.count == 1 - ? foreignKeysByTableName[T.tableName]?.first - : nil + ? foreignKeysByTableName[T.tableName]?.first + : nil + ) + } else { + reportIssue( + """ + Local database record could not be found for '\(serverRecord.recordID.recordName)'. + """ ) - return columnNames } + columnNames = _columnNames } else { columnNames = T.TableColumns.writableColumns.map(\.name) } - try await userDatabase.write { db in - do { - try #sql(upsert(T.self, record: serverRecord, columnNames: columnNames)).execute(db) - try UnsyncedRecordID.find(serverRecord.recordID).delete().execute(db) - try SyncMetadata - .where { $0.recordName.eq(serverRecord.recordID.recordName) } - .update { $0.setLastKnownServerRecord(serverRecord) } - .execute(db) - } catch { - guard - let error = error as? DatabaseError, - error.resultCode == .SQLITE_CONSTRAINT, - error.extendedResultCode == .SQLITE_CONSTRAINT_FOREIGNKEY - else { - throw error - } - try UnsyncedRecordID.insert(or: .ignore) { - UnsyncedRecordID(recordID: serverRecord.recordID) - } + do { + try #sql(upsert(T.self, record: serverRecord, columnNames: columnNames)).execute(db) + try UnsyncedRecordID.find(serverRecord.recordID).delete().execute(db) + try SyncMetadata + .where { $0.recordName.eq(serverRecord.recordID.recordName) } + .update { $0.setLastKnownServerRecord(serverRecord) } .execute(db) + } catch { + guard + let error = error as? DatabaseError, + error.resultCode == .SQLITE_CONSTRAINT, + error.extendedResultCode == .SQLITE_CONSTRAINT_FOREIGNKEY + else { + throw error + } + try UnsyncedRecordID.insert(or: .ignore) { + UnsyncedRecordID(recordID: serverRecord.recordID) } + .execute(db) } } - try await open(table) + try open(table) } } From 522fa1c995d12134df2a1288de7a0ac50e88b3ab Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Sun, 14 Sep 2025 20:24:53 -0500 Subject: [PATCH 2/5] wip --- Sources/SQLiteData/CloudKit/SyncEngine.swift | 42 +++++++++++--------- 1 file changed, 23 insertions(+), 19 deletions(-) diff --git a/Sources/SQLiteData/CloudKit/SyncEngine.swift b/Sources/SQLiteData/CloudKit/SyncEngine.swift index 76951e90..93865a0c 100644 --- a/Sources/SQLiteData/CloudKit/SyncEngine.swift +++ b/Sources/SQLiteData/CloudKit/SyncEngine.swift @@ -1259,24 +1259,27 @@ case share(CKShare) case reference(CKShare.Reference) } - let shares = LockIsolated<[ShareOrReference]>([]) - await withErrorReporting(.sqliteDataCloudKitFailure) { - try await userDatabase.write { db in - for record in modifications { - if let share = record as? CKShare { - shares.withValue { $0.append(.share(share)) } - } else { - upsertFromServerRecord(record, db: db) - if let shareReference = record.share { - shares.withValue { $0.append(.reference(shareReference)) } + let shares: [ShareOrReference] = + await withErrorReporting(.sqliteDataCloudKitFailure) { + try await userDatabase.write { db in + var shares: [ShareOrReference] = [] + for record in modifications { + if let share = record as? CKShare { + shares.append(.share(share)) + } else { + upsertFromServerRecord(record, db: db) + if let shareReference = record.share { + shares.append(.reference(shareReference)) + } } } + return shares } } - } + ?? [] await withTaskGroup(of: Void.self) { group in - for share in shares.withValue(\.self) { + for share in shares { group.addTask { switch share { case .share(let share): @@ -1535,11 +1538,12 @@ return } - let metadata = try SyncMetadata + let metadata = + try SyncMetadata .where { $0.recordName.eq(serverRecord.recordID.recordName) } .fetchOne(db) serverRecord.userModificationDate = - metadata?.userModificationDate ?? serverRecord.userModificationDate + metadata?.userModificationDate ?? serverRecord.userModificationDate func open(_: T.Type) throws { let columnNames: [String] @@ -1552,14 +1556,14 @@ row: T(queryOutput: row), columnNames: &_columnNames, parentForeignKey: foreignKeysByTableName[T.tableName]?.count == 1 - ? foreignKeysByTableName[T.tableName]?.first - : nil + ? foreignKeysByTableName[T.tableName]?.first + : nil ) } else { reportIssue( - """ - Local database record could not be found for '\(serverRecord.recordID.recordName)'. - """ + """ + Local database record could not be found for '\(serverRecord.recordID.recordName)'. + """ ) } columnNames = _columnNames From 10f233b2247691b1a104449333ee701ad1a8b3d7 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Mon, 15 Sep 2025 12:12:48 -0500 Subject: [PATCH 3/5] wip --- Sources/SQLiteData/CloudKit/SyncEngine.swift | 8 ++++---- Tests/SQLiteDataTests/Internal/BaseCloudKitTests.swift | 4 ++-- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/Sources/SQLiteData/CloudKit/SyncEngine.swift b/Sources/SQLiteData/CloudKit/SyncEngine.swift index 14c8672c..f314427d 100644 --- a/Sources/SQLiteData/CloudKit/SyncEngine.swift +++ b/Sources/SQLiteData/CloudKit/SyncEngine.swift @@ -1203,9 +1203,9 @@ for (recordType, recordIDs) in deletedRecordIDsByRecordType { let recordPrimaryKeys = recordIDs.compactMap(\.recordPrimaryKey) if let table = tablesByName[recordType] { - func open(_: T.Type) { - withErrorReporting(.sqliteDataCloudKitFailure) { - try userDatabase.write { db in + func open(_: T.Type) async { + await withErrorReporting(.sqliteDataCloudKitFailure) { + try await userDatabase.write { db in try T .where { $0.primaryKey.in( @@ -1222,7 +1222,7 @@ } } } - open(table) + await open(table) } else if recordType == CKRecord.SystemType.share { for recordID in recordIDs { await withErrorReporting(.sqliteDataCloudKitFailure) { diff --git a/Tests/SQLiteDataTests/Internal/BaseCloudKitTests.swift b/Tests/SQLiteDataTests/Internal/BaseCloudKitTests.swift index 97d8dc5f..e0c7176b 100644 --- a/Tests/SQLiteDataTests/Internal/BaseCloudKitTests.swift +++ b/Tests/SQLiteDataTests/Internal/BaseCloudKitTests.swift @@ -155,8 +155,8 @@ extension SyncEngine { convenience init( container: any CloudContainer, userDatabase: UserDatabase, - tables: [any PrimaryKeyedTable.Type], - privateTables: [any PrimaryKeyedTable.Type] = [], + tables: [any (PrimaryKeyedTable & _SendableMetatype).Type], + privateTables: [any (PrimaryKeyedTable & _SendableMetatype).Type] = [], startImmediately: Bool = true ) async throws { try self.init( From de6fda102b0b0624c4884877304510a9952bb1ab Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Mon, 15 Sep 2025 13:06:08 -0500 Subject: [PATCH 4/5] wip --- Sources/SQLiteData/CloudKit/SyncEngine.swift | 1 - 1 file changed, 1 deletion(-) diff --git a/Sources/SQLiteData/CloudKit/SyncEngine.swift b/Sources/SQLiteData/CloudKit/SyncEngine.swift index 72a77aaa..6d035ae5 100644 --- a/Sources/SQLiteData/CloudKit/SyncEngine.swift +++ b/Sources/SQLiteData/CloudKit/SyncEngine.swift @@ -1571,7 +1571,6 @@ try SyncMetadata .where { $0.recordName.eq(serverRecord.recordID.recordName) } .fetchOne(db) - serverRecord.userModificationDate = serverRecord.userModificationTime = metadata?.userModificationTime ?? serverRecord.userModificationTime From 9d2dc8e0a487f7065c223c713c154d2b25f458c1 Mon Sep 17 00:00:00 2001 From: Brandon Williams Date: Mon, 15 Sep 2025 13:07:44 -0500 Subject: [PATCH 5/5] wip --- .../SQLiteDataTests/CloudKitTests/SharingTests.swift | 12 ++++++------ .../CloudKitTests/SyncEngineLifecycleTests.swift | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/Tests/SQLiteDataTests/CloudKitTests/SharingTests.swift b/Tests/SQLiteDataTests/CloudKitTests/SharingTests.swift index e89ca6b5..ca4e414a 100644 --- a/Tests/SQLiteDataTests/CloudKitTests/SharingTests.swift +++ b/Tests/SQLiteDataTests/CloudKitTests/SharingTests.swift @@ -383,7 +383,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: true, │ │ isShared: false, │ - │ userModificationDate: Date(1970-01-01T00:00:00.000Z) │ + │ userModificationTime: 0 │ │ ) │ ├─────────────────────────────────────────────────────────────────────────────────────────┤ │ SyncMetadata( │ @@ -412,7 +412,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: true, │ │ isShared: false, │ - │ userModificationDate: Date(1970-01-01T00:01:00.000Z) │ + │ userModificationTime: 60 │ │ ) │ ├─────────────────────────────────────────────────────────────────────────────────────────┤ │ SyncMetadata( │ @@ -441,7 +441,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: true, │ │ isShared: false, │ - │ userModificationDate: Date(1970-01-01T00:01:00.000Z) │ + │ userModificationTime: 60 │ │ ) │ └─────────────────────────────────────────────────────────────────────────────────────────┘ """ @@ -1105,7 +1105,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: true, │ │ isShared: false, │ - │ userModificationDate: Date(1970-01-01T00:00:00.000Z) │ + │ userModificationTime: 0 │ │ ) │ ├──────────────────────────────────────────────────────────────────────────────────────────────┤ │ SyncMetadata( │ @@ -1134,7 +1134,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: true, │ │ isShared: false, │ - │ userModificationDate: Date(1970-01-01T00:00:00.000Z) │ + │ userModificationTime: 0 │ │ ) │ ├──────────────────────────────────────────────────────────────────────────────────────────────┤ │ SyncMetadata( │ @@ -1167,7 +1167,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: true, │ │ isShared: true, │ - │ userModificationDate: Date(1970-01-01T00:00:00.000Z) │ + │ userModificationTime: 0 │ │ ) │ └──────────────────────────────────────────────────────────────────────────────────────────────┘ """ diff --git a/Tests/SQLiteDataTests/CloudKitTests/SyncEngineLifecycleTests.swift b/Tests/SQLiteDataTests/CloudKitTests/SyncEngineLifecycleTests.swift index cce9b97f..76784d08 100644 --- a/Tests/SQLiteDataTests/CloudKitTests/SyncEngineLifecycleTests.swift +++ b/Tests/SQLiteDataTests/CloudKitTests/SyncEngineLifecycleTests.swift @@ -296,7 +296,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: true, │ │ isShared: false, │ - │ userModificationDate: Date(1970-01-01T00:00:00.000Z) │ + │ userModificationTime: 0 │ │ ) │ ├───────────────────────────────────────────────────────────────────────────┤ │ SyncMetadata( │ @@ -312,7 +312,7 @@ │ _isDeleted: false, │ │ hasLastKnownServerRecord: false, │ │ isShared: false, │ - │ userModificationDate: Date(1970-01-01T00:01:00.000Z) │ + │ userModificationTime: 60 │ │ ) │ └───────────────────────────────────────────────────────────────────────────┘ """