Skip to content

More efficient joins to SyncMetadata table #163

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephencelis merged 7 commits into from
Sep 8, 2025
Merged

More efficient joins to SyncMetadata table #163

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
dc43b62
Improve tools for joining SyncMetadata table.
mbrandonw Sep 6, 2025
db76cb6
wip
mbrandonw Sep 6, 2025
9946927
wip
stephencelis Sep 8, 2025
8c275a5
wip
stephencelis Sep 8, 2025
27e0eee
wip
stephencelis Sep 8, 2025
333bb81
wip
stephencelis Sep 8, 2025
4dfe25e
wip
stephencelis Sep 8, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ jobs:
- uses: actions/checkout@v4
- name: Select Xcode ${{ matrix.xcode }}
run: sudo xcode-select -s /Applications/Xcode_${{ matrix.xcode }}.app
- name: List devices available
run: xcrun simctl list --json devices available 'iPhone'
- name: xcodebuild ${{ matrix.scheme }}
run: make DERIVED_DATA_PATH=~/.derivedData SCHEME="${{ matrix.scheme }}" xcodebuild-raw
- name: Output test failures
Expand Down
6 changes: 2 additions & 4 deletions Examples/Reminders/RemindersLists.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,8 @@ class RemindersListsModel {
RemindersList
.group(by: \.id)
.order(by: \.position)
.leftJoin(Reminder.all) {
$0.id.eq($1.remindersListID) && !$1.isCompleted
}
.leftJoin(SyncMetadata.all) { $0.recordName.eq($2.recordName) }
.leftJoin(Reminder.all) { $0.id.eq($1.remindersListID) && !$1.isCompleted }
.leftJoin(SyncMetadata.all) { $0.hasMetadata(in: $2) }
.select {
ReminderListState.Columns(
remindersCount: $1.id.count(),
Expand Down
14 changes: 7 additions & 7 deletions Sources/SQLiteData/CloudKit/Internal/Metadatabase.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@
migrator.registerMigration("Create Metadata Tables") { db in
try #sql(
"""
CREATE TABLE IF NOT EXISTS "\(raw: .sqliteDataCloudKitSchemaName)_metadata" (
CREATE TABLE "\(raw: .sqliteDataCloudKitSchemaName)_metadata" (
"recordPrimaryKey" TEXT NOT NULL,
"recordType" TEXT NOT NULL,
"recordName" TEXT NOT NULL AS ("recordPrimaryKey" || ':' || "recordType"),
Expand All @@ -73,21 +73,21 @@
.execute(db)
try #sql(
"""
CREATE INDEX IF NOT EXISTS "\(raw: .sqliteDataCloudKitSchemaName)_metadata_parentRecordName"
CREATE INDEX "\(raw: .sqliteDataCloudKitSchemaName)_metadata_parentRecordName"
ON "\(raw: .sqliteDataCloudKitSchemaName)_metadata"("parentRecordName")
"""
)
.execute(db)
try #sql(
"""
CREATE INDEX IF NOT EXISTS "\(raw: .sqliteDataCloudKitSchemaName)_metadata_isShared"
CREATE INDEX "\(raw: .sqliteDataCloudKitSchemaName)_metadata_isShared"
ON "\(raw: .sqliteDataCloudKitSchemaName)_metadata"("isShared")
"""
)
.execute(db)
try #sql(
"""
CREATE TABLE IF NOT EXISTS "\(raw: .sqliteDataCloudKitSchemaName)_recordTypes" (
CREATE TABLE "\(raw: .sqliteDataCloudKitSchemaName)_recordTypes" (
"tableName" TEXT NOT NULL PRIMARY KEY,
"schema" TEXT NOT NULL,
"tableInfo" TEXT NOT NULL
Expand All @@ -97,7 +97,7 @@
.execute(db)
try #sql(
"""
CREATE TABLE IF NOT EXISTS "\(raw: .sqliteDataCloudKitSchemaName)_stateSerialization" (
CREATE TABLE "\(raw: .sqliteDataCloudKitSchemaName)_stateSerialization" (
"scope" TEXT NOT NULL PRIMARY KEY,
"data" TEXT NOT NULL
) STRICT
Expand All @@ -106,7 +106,7 @@
.execute(db)
try #sql(
"""
CREATE TABLE IF NOT EXISTS "\(raw: .sqliteDataCloudKitSchemaName)_unsyncedRecordIDs" (
CREATE TABLE "\(raw: .sqliteDataCloudKitSchemaName)_unsyncedRecordIDs" (
"recordName" TEXT NOT NULL,
"zoneName" TEXT NOT NULL,
"ownerName" TEXT NOT NULL,
Expand All @@ -117,7 +117,7 @@
.execute(db)
try #sql(
"""
CREATE TABLE IF NOT EXISTS "\(raw: .sqliteDataCloudKitSchemaName)_pendingRecordZoneChanges" (
CREATE TABLE "\(raw: .sqliteDataCloudKitSchemaName)_pendingRecordZoneChanges" (
"pendingRecordZoneChange" BLOB NOT NULL
) STRICT
"""
Expand Down
13 changes: 11 additions & 2 deletions Sources/SQLiteData/CloudKit/SyncMetadata.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -163,8 +163,17 @@

@available(iOS 17, macOS 14, tvOS 17, watchOS 10, *)
extension PrimaryKeyedTableDefinition where PrimaryKey.QueryOutput: IdentifierStringConvertible {
public var recordName: some QueryExpression<String> {
_recordName
Comment on lines -166 to -167
Copy link
Copy Markdown
Member Author

@mbrandonw mbrandonw Sep 6, 2025

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I don't think we should publicly expose this at all, it's too dangerous. We still have the internal _recordName one which we use for selects, which is fine.

/// A query expression for whether or not this row has associated sync metadata.
///
/// This helper can be useful when joining your tables to the ``SyncMetadata`` table:
///
/// ```swift
/// RemindersList
/// .leftJoin(SyncMetadata.all) { $0.hasMetadata.in($1) }
/// ```
public func hasMetadata(in metadata: SyncMetadata.TableColumns) -> some QueryExpression<Bool> {
metadata.recordType.eq(QueryValue.tableName)
&& #sql("\(primaryKey)").eq(metadata.recordPrimaryKey)
}
}

Expand Down
6 changes: 3 additions & 3 deletions Sources/SQLiteData/Documentation.docc/Articles/CloudKit.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -571,7 +571,7 @@ following:

@FetchAll(
RemindersList
.leftJoin(SyncMetadata.all) { $0.recordName.eq($1.recordName) }
.leftJoin(SyncMetadata.all) { $0.hasMetadata(in: $1) }
.select {
Row.Columns(
remindersList: $0,
Expand All @@ -582,8 +582,8 @@ following:
var rows
```

Here we have used the ``StructuredQueriesCore/PrimaryKeyedTableDefinition/recordName`` helper that
is defined on all primary key tables so that we can join ``SyncMetadata`` to `RemindersList`.
Here we have used the ``StructuredQueriesCore/PrimaryKeyedTableDefinition/hasMetadata(in:)`` helper
that is defined on all primary key tables so that we can join ``SyncMetadata`` to `RemindersList`.

<!--
## How SQLiteData handles distributed schema scenarios
Expand Down
1 change: 1 addition & 0 deletions Sources/SQLiteData/Documentation.docc/SQLiteData.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -305,4 +305,5 @@ with SQLite to take full advantage of GRDB and SQLiteData.
- ``Dependencies/DependencyValues/defaultSyncEngine``
- ``IdentifierStringConvertible``
- ``SyncMetadata``
- ``StructuredQueriesCore/PrimaryKeyedTableDefinition/hasMetadata(in:)``
- ``SharedRecord``
76 changes: 76 additions & 0 deletions Tests/SQLiteDataTests/CloudKitTests/MetadataTests.swift
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -488,6 +488,82 @@
"""
}
}

@available(iOS 17, macOS 14, tvOS 17, watchOS 10, *)
@Test func hasMetadataHelper() async throws {
try await userDatabase.userWrite { db in
try db.seed {
RemindersList(id: 1, title: "Personal")
RemindersList(id: 2, title: "Work")
Reminder(id: 1, title: "Groceries", remindersListID: 1)
}
}

try await syncEngine.processPendingRecordZoneChanges(scope: .private)

assertQuery(
RemindersList.join(SyncMetadata.all) { $0.hasMetadata(in: $1) },
database: userDatabase.database
) {
"""
┌─────────────────────┬────────────────────────────────────────────────────────────────────┐
│ RemindersList( │ SyncMetadata( │
│ id: 1, │ recordPrimaryKey: "1", │
│ title: "Personal" │ recordType: "remindersLists", │
│ ) │ recordName: "1:remindersLists", │
│ │ parentRecordPrimaryKey: nil, │
│ │ parentRecordType: nil, │
│ │ parentRecordName: nil, │
│ │ lastKnownServerRecord: CKRecord( │
│ │ recordID: CKRecord.ID(1:remindersLists/zone/__defaultOwner__), │
│ │ recordType: "remindersLists", │
│ │ parent: nil, │
│ │ share: nil │
│ │ ), │
│ │ _lastKnownServerRecordAllFields: CKRecord( │
│ │ recordID: CKRecord.ID(1:remindersLists/zone/__defaultOwner__), │
│ │ recordType: "remindersLists", │
│ │ parent: nil, │
│ │ share: nil, │
│ │ id: 1, │
│ │ title: "Personal" │
│ │ ), │
│ │ share: nil, │
│ │ _isDeleted: false, │
│ │ isShared: false, │
│ │ userModificationDate: Date(1970-01-01T00:00:00.000Z) │
│ │ ) │
├─────────────────────┼────────────────────────────────────────────────────────────────────┤
│ RemindersList( │ SyncMetadata( │
│ id: 2, │ recordPrimaryKey: "2", │
│ title: "Work" │ recordType: "remindersLists", │
│ ) │ recordName: "2:remindersLists", │
│ │ parentRecordPrimaryKey: nil, │
│ │ parentRecordType: nil, │
│ │ parentRecordName: nil, │
│ │ lastKnownServerRecord: CKRecord( │
│ │ recordID: CKRecord.ID(2:remindersLists/zone/__defaultOwner__), │
│ │ recordType: "remindersLists", │
│ │ parent: nil, │
│ │ share: nil │
│ │ ), │
│ │ _lastKnownServerRecordAllFields: CKRecord( │
│ │ recordID: CKRecord.ID(2:remindersLists/zone/__defaultOwner__), │
│ │ recordType: "remindersLists", │
│ │ parent: nil, │
│ │ share: nil, │
│ │ id: 2, │
│ │ title: "Work" │
│ │ ), │
│ │ share: nil, │
│ │ _isDeleted: false, │
│ │ isShared: false, │
│ │ userModificationDate: Date(1970-01-01T00:00:00.000Z) │
│ │ ) │
└─────────────────────┴────────────────────────────────────────────────────────────────────┘
"""
}
}
}
}
#endif
0 ...dKitTests/UnattachedSyncEngineTests.swift → ...dKitTests/UnattachedSyncEngineTests.swift
View file
Open in desktop
File renamed without changes.
Loading