From df8d68cdeffebb08041352f45d0ef526105dbb02 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Tue, 1 Aug 2023 14:33:35 -0700 Subject: [PATCH 1/2] Track rows read and written within sqlite. This takes the row-counting logic from libsql[1] and sticks it into our copy of SQLite. libsql is an ambitious fork of SQLite, so switching out the entire library would be quite risky. Porting the row-counting logic from libsql (which is MIT-licensed) is a much less risky change, though it does require us to carry a patch around forever, or at least until SQLite implements something similar, if they ever do. [1] A fork of sqlite that accepts contributions. https://github.com/libsql/libsql --- WORKSPACE | 4 + .../sqlite/0001-row-counts-amalgamation.patch | 302 ++++++++++++++++++ patches/sqlite/0001-row-counts-plain.patch | 251 +++++++++++++++ patches/sqlite/README | 69 ++++ 4 files changed, 626 insertions(+) create mode 100644 patches/sqlite/0001-row-counts-amalgamation.patch create mode 100644 patches/sqlite/0001-row-counts-plain.patch create mode 100644 patches/sqlite/README diff --git a/WORKSPACE b/WORKSPACE index f1b05575eba..587c11ef975 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -46,6 +46,10 @@ http_archive( strip_prefix = "sqlite-amalgamation-3400100", type = "zip", url = "https://sqlite.org/2022/sqlite-amalgamation-3400100.zip", + patches = [ + "//:patches/sqlite/0001-row-counts-amalgamation.patch", + ], + patch_args = ["-p1"], ) http_archive( diff --git a/patches/sqlite/0001-row-counts-amalgamation.patch b/patches/sqlite/0001-row-counts-amalgamation.patch new file mode 100644 index 00000000000..50d41a2221d --- /dev/null +++ b/patches/sqlite/0001-row-counts-amalgamation.patch @@ -0,0 +1,302 @@ +diff -u5 -r sqlite-amalgamation-pristine/shell.c sqlite-amalgamation-modified/shell.c +--- sqlite-amalgamation-pristine/shell.c 2022-12-28 06:26:39.000000000 -0800 ++++ sqlite-amalgamation-modified/shell.c 2023-08-04 14:53:19.663535828 -0700 +@@ -17356,10 +17356,15 @@ + raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); + raw_printf(pArg->out, "Number of times run: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); + raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); ++ ++ iCur = sqlite3_stmt_status(pArg->pStmt, LIBSQL_STMTSTATUS_ROWS_READ, bReset); ++ raw_printf(pArg->out, "Rows read: %d\n", iCur); ++ iCur = sqlite3_stmt_status(pArg->pStmt, LIBSQL_STMTSTATUS_ROWS_WRITTEN, bReset); ++ raw_printf(pArg->out, "Rows written: %d\n", iCur); + } + + #ifdef __linux__ + displayLinuxIoStats(pArg->out); + #endif +diff -u5 -r sqlite-amalgamation-pristine/sqlite3.c sqlite-amalgamation-modified/sqlite3.c +--- sqlite-amalgamation-pristine/sqlite3.c 2022-12-28 06:26:39.000000000 -0800 ++++ sqlite-amalgamation-modified/sqlite3.c 2023-08-04 14:53:19.667535836 -0700 +@@ -452,11 +452,11 @@ + ** [sqlite3_libversion_number()], [sqlite3_sourceid()], + ** [sqlite_version()] and [sqlite_source_id()]. + */ + #define SQLITE_VERSION "3.40.1" + #define SQLITE_VERSION_NUMBER 3040001 +-#define SQLITE_SOURCE_ID "2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f38a24" ++#define SQLITE_SOURCE_ID "2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f3alt1" + + /* + ** CAPI3REF: Run-Time Library Version Numbers + ** KEYWORDS: sqlite3_version sqlite3_sourceid + ** +@@ -8921,10 +8921,18 @@ + ** used to store the prepared statement. ^This value is not actually + ** a counter, and so the resetFlg parameter to sqlite3_stmt_status() + ** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. + ** + ** ++** ++** [[LIBSQL_STMTSTATUS_ROWS_READ]] ++** [[LIBSQL_STMTSTATUS_ROWS_WRITTEN]] ++**
LIBSQL_STMTSTATUS_ROWS_READ
++** LIBSQL_STMTSTATUS_ROWS_WRITTEN
++**
^LIBSQL_STMTSTATUS_ROWS_READ is the number of rows read when executing ++** this statement. LIBSQL_STMTSTATUS_ROWS_WRITTEN value is the number of ++** rows written. + */ + #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 + #define SQLITE_STMTSTATUS_SORT 2 + #define SQLITE_STMTSTATUS_AUTOINDEX 3 + #define SQLITE_STMTSTATUS_VM_STEP 4 +@@ -8932,10 +8940,14 @@ + #define SQLITE_STMTSTATUS_RUN 6 + #define SQLITE_STMTSTATUS_FILTER_MISS 7 + #define SQLITE_STMTSTATUS_FILTER_HIT 8 + #define SQLITE_STMTSTATUS_MEMUSED 99 + ++#define LIBSQL_STMTSTATUS_BASE 1024 ++#define LIBSQL_STMTSTATUS_ROWS_READ LIBSQL_STMTSTATUS_BASE + 1 ++#define LIBSQL_STMTSTATUS_ROWS_WRITTEN LIBSQL_STMTSTATUS_BASE + 2 ++ + /* + ** CAPI3REF: Custom Page Cache Object + ** + ** The sqlite3_pcache type is opaque. It is implemented by + ** the pluggable module. The SQLite core has no knowledge of +@@ -22778,10 +22790,11 @@ + bft readOnly:1; /* True for statements that do not write */ + bft bIsReader:1; /* True for statements that read */ + yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ + yDbMask lockMask; /* Subset of btreeMask that requires a lock */ + u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */ ++ u32 aLibsqlCounter[3]; /* libSQL extension: Counters used by sqlite3_stmt_status()*/ + char *zSql; /* Text of the SQL statement that generated this */ + #ifdef SQLITE_ENABLE_NORMALIZE + char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ + #endif +@@ -89160,11 +89173,11 @@ + SQLITE_API int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ + Vdbe *pVdbe = (Vdbe*)pStmt; + u32 v; + #ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt +- || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter))) ++ || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||(op>=ArraySize(pVdbe->aCounter)&&oplookaside.pEnd = db->lookaside.pStart; + sqlite3VdbeDelete(pVdbe); + db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; + sqlite3_mutex_leave(db->mutex); ++ }else if( op>=LIBSQL_STMTSTATUS_BASE ){ ++ v = pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE]; ++ if( resetFlag ) pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE] = 0; + }else{ + v = pVdbe->aCounter[op]; + if( resetFlag ) pVdbe->aCounter[op] = 0; + } + return (int)v; +@@ -93285,10 +93301,11 @@ + if( pOp->p3 ){ + nEntry = sqlite3BtreeRowCountEst(pCrsr); + }else{ + nEntry = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3BtreeCount(db, pCrsr, &nEntry); ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE] += nEntry; + if( rc ) goto abort_due_to_error; + } + pOut = out2Prerelease(p, pOp); + pOut->u.i = nEntry; + goto check_for_interrupt; +@@ -94444,10 +94461,11 @@ + if( eqOnly && r.eqSeen==0 ){ + assert( res!=0 ); + goto seek_not_found; + } + } ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + #ifdef SQLITE_TEST + sqlite3_search_count++; + #endif + if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); + if( res<0 || (res==0 && oc==OP_SeekGT) ){ +@@ -95012,10 +95030,11 @@ + pC->nullRow = 0; + pC->cacheStatus = CACHE_STALE; + pC->deferredMoveto = 0; + VdbeBranchTaken(res!=0,2); + pC->seekResult = res; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + if( res!=0 ){ + assert( rc==SQLITE_OK ); + if( pOp->p2==0 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ +@@ -95268,10 +95287,11 @@ + } + } + if( pOp->p5 & OPFLAG_ISNOOP ) break; + #endif + ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); + x.pData = pData->z; + x.nData = pData->n; +@@ -95451,10 +95471,11 @@ + pC->seekResult = 0; + if( rc ) goto abort_due_to_error; + + /* Invoke the update-hook if required. */ + if( opflags & OPFLAG_NCHANGE ){ ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + p->nChange++; + if( db->xUpdateCallback && ALWAYS(pTab!=0) && HasRowid(pTab) ){ + db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName, + pC->movetoTarget); + assert( pC->iDb>=0 ); +@@ -95738,10 +95759,11 @@ + rc = sqlite3BtreeLast(pCrsr, &res); + pC->nullRow = (u8)res; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + if( rc ) goto abort_due_to_error; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + } + break; +@@ -95842,10 +95864,11 @@ + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + } + if( rc ) goto abort_due_to_error; + pC->nullRow = (u8)res; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + assert( pOp->p2>0 && pOp->p2nOp ); + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; + } +@@ -95946,10 +95969,11 @@ + pC->cacheStatus = CACHE_STALE; + VdbeBranchTaken(rc==SQLITE_OK,2); + if( rc==SQLITE_OK ){ + pC->nullRow = 0; + p->aCounter[pOp->p5]++; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + #ifdef SQLITE_TEST + sqlite3_search_count++; + #endif + goto jump_to_p2_and_check_for_interrupt; + } +@@ -95997,10 +96021,11 @@ + assert( pC!=0 ); + assert( !isSorter(pC) ); + pIn2 = &aMem[pOp->p2]; + assert( (pIn2->flags & MEM_Blob) || (pOp->p5 & OPFLAG_PREFORMAT) ); + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc ) goto abort_due_to_error; + x.nKey = pIn2->n; +@@ -96397,10 +96422,11 @@ + assert( p->readOnly==0 ); + assert( DbMaskTest(p->btreeMask, pOp->p2) ); + rc = sqlite3BtreeClearTable(db->aDb[pOp->p2].pBt, (u32)pOp->p1, &nChange); + if( pOp->p3 ){ + p->nChange += nChange; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE] += nChange; + if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); + aMem[pOp->p3].u.i += nChange; + } +@@ -97879,10 +97905,11 @@ + ** some other method is next invoked on the save virtual table cursor. + */ + rc = pModule->xNext(pCur->uc.pVCur); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + res = pModule->xEof(pCur->uc.pVCur); + VdbeBranchTaken(!res,2); + if( !res ){ + /* If there is data, jump to P2 */ + goto jump_to_p2_and_check_for_interrupt; +@@ -98000,10 +98027,11 @@ + rc = SQLITE_OK; + }else{ + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); + } + }else{ ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + p->nChange++; + } + if( rc ) goto abort_due_to_error; + } + break; +diff -u5 -r sqlite-amalgamation-pristine/sqlite3.h sqlite-amalgamation-modified/sqlite3.h +--- sqlite-amalgamation-pristine/sqlite3.h 2022-12-28 06:26:39.000000000 -0800 ++++ sqlite-amalgamation-modified/sqlite3.h 2023-08-04 14:53:19.667535836 -0700 +@@ -146,11 +146,11 @@ + ** [sqlite3_libversion_number()], [sqlite3_sourceid()], + ** [sqlite_version()] and [sqlite_source_id()]. + */ + #define SQLITE_VERSION "3.40.1" + #define SQLITE_VERSION_NUMBER 3040001 +-#define SQLITE_SOURCE_ID "2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f38a24" ++#define SQLITE_SOURCE_ID "2022-12-28 14:03:47 df5c253c0b3dd24916e4ec7cf77d3db5294cc9fd45ae7b9c5e82ad8197f3alt1" + + /* + ** CAPI3REF: Run-Time Library Version Numbers + ** KEYWORDS: sqlite3_version sqlite3_sourceid + ** +@@ -8615,10 +8615,18 @@ + ** used to store the prepared statement. ^This value is not actually + ** a counter, and so the resetFlg parameter to sqlite3_stmt_status() + ** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. + **
+ ** ++** ++** [[LIBSQL_STMTSTATUS_ROWS_READ]] ++** [[LIBSQL_STMTSTATUS_ROWS_WRITTEN]] ++**
LIBSQL_STMTSTATUS_ROWS_READ
++** LIBSQL_STMTSTATUS_ROWS_WRITTEN
++**
^LIBSQL_STMTSTATUS_ROWS_READ is the number of rows read when executing ++** this statement. LIBSQL_STMTSTATUS_ROWS_WRITTEN value is the number of ++** rows written. + */ + #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 + #define SQLITE_STMTSTATUS_SORT 2 + #define SQLITE_STMTSTATUS_AUTOINDEX 3 + #define SQLITE_STMTSTATUS_VM_STEP 4 +@@ -8626,10 +8634,14 @@ + #define SQLITE_STMTSTATUS_RUN 6 + #define SQLITE_STMTSTATUS_FILTER_MISS 7 + #define SQLITE_STMTSTATUS_FILTER_HIT 8 + #define SQLITE_STMTSTATUS_MEMUSED 99 + ++#define LIBSQL_STMTSTATUS_BASE 1024 ++#define LIBSQL_STMTSTATUS_ROWS_READ LIBSQL_STMTSTATUS_BASE + 1 ++#define LIBSQL_STMTSTATUS_ROWS_WRITTEN LIBSQL_STMTSTATUS_BASE + 2 ++ + /* + ** CAPI3REF: Custom Page Cache Object + ** + ** The sqlite3_pcache type is opaque. It is implemented by + ** the pluggable module. The SQLite core has no knowledge of diff --git a/patches/sqlite/0001-row-counts-plain.patch b/patches/sqlite/0001-row-counts-plain.patch new file mode 100644 index 00000000000..e814209692a --- /dev/null +++ b/patches/sqlite/0001-row-counts-plain.patch @@ -0,0 +1,251 @@ +# This patch contains code from libsql, which is released under the MIT license. +# +# See https://github.com/libsql/libsql for details. +diff -u5 -r sqlite-src-3400100-pristine/src/shell.c.in sqlite-src-3400100-modified/src/shell.c.in +--- sqlite-src-3400100-pristine/src/shell.c.in 2023-08-03 14:15:24.760062869 -0700 ++++ sqlite-src-3400100-modified/src/shell.c.in 2023-08-03 14:15:53.042646929 -0700 +@@ -2952,10 +2952,15 @@ + raw_printf(pArg->out, "Reprepare operations: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_RUN, bReset); + raw_printf(pArg->out, "Number of times run: %d\n", iCur); + iCur = sqlite3_stmt_status(pArg->pStmt, SQLITE_STMTSTATUS_MEMUSED, bReset); + raw_printf(pArg->out, "Memory used by prepared stmt: %d\n", iCur); ++ ++ iCur = sqlite3_stmt_status(pArg->pStmt, LIBSQL_STMTSTATUS_ROWS_READ, bReset); ++ raw_printf(pArg->out, "Rows read: %d\n", iCur); ++ iCur = sqlite3_stmt_status(pArg->pStmt, LIBSQL_STMTSTATUS_ROWS_WRITTEN, bReset); ++ raw_printf(pArg->out, "Rows written: %d\n", iCur); + } + + #ifdef __linux__ + displayLinuxIoStats(pArg->out); + #endif +diff -u5 -r sqlite-src-3400100-pristine/src/sqlite.h.in sqlite-src-3400100-modified/src/sqlite.h.in +--- sqlite-src-3400100-pristine/src/sqlite.h.in 2022-12-28 06:26:33.000000000 -0800 ++++ sqlite-src-3400100-modified/src/sqlite.h.in 2023-08-03 14:18:49.490588655 -0700 +@@ -8615,10 +8615,18 @@ + ** used to store the prepared statement. ^This value is not actually + ** a counter, and so the resetFlg parameter to sqlite3_stmt_status() + ** is ignored when the opcode is SQLITE_STMTSTATUS_MEMUSED. + **
+ ** ++** ++** [[LIBSQL_STMTSTATUS_ROWS_READ]] ++** [[LIBSQL_STMTSTATUS_ROWS_WRITTEN]] ++**
LIBSQL_STMTSTATUS_ROWS_READ
++** LIBSQL_STMTSTATUS_ROWS_WRITTEN
++**
^LIBSQL_STMTSTATUS_ROWS_READ is the number of rows read when executing ++** this statement. LIBSQL_STMTSTATUS_ROWS_WRITTEN value is the number of ++** rows written. + */ + #define SQLITE_STMTSTATUS_FULLSCAN_STEP 1 + #define SQLITE_STMTSTATUS_SORT 2 + #define SQLITE_STMTSTATUS_AUTOINDEX 3 + #define SQLITE_STMTSTATUS_VM_STEP 4 +@@ -8626,10 +8634,14 @@ + #define SQLITE_STMTSTATUS_RUN 6 + #define SQLITE_STMTSTATUS_FILTER_MISS 7 + #define SQLITE_STMTSTATUS_FILTER_HIT 8 + #define SQLITE_STMTSTATUS_MEMUSED 99 + ++#define LIBSQL_STMTSTATUS_BASE 1024 ++#define LIBSQL_STMTSTATUS_ROWS_READ LIBSQL_STMTSTATUS_BASE + 1 ++#define LIBSQL_STMTSTATUS_ROWS_WRITTEN LIBSQL_STMTSTATUS_BASE + 2 ++ + /* + ** CAPI3REF: Custom Page Cache Object + ** + ** The sqlite3_pcache type is opaque. It is implemented by + ** the pluggable module. The SQLite core has no knowledge of +diff -u5 -r sqlite-src-3400100-pristine/src/vdbeapi.c sqlite-src-3400100-modified/src/vdbeapi.c +--- sqlite-src-3400100-pristine/src/vdbeapi.c 2022-12-28 06:26:33.000000000 -0800 ++++ sqlite-src-3400100-modified/src/vdbeapi.c 2023-08-03 14:19:40.916453104 -0700 +@@ -1826,11 +1826,11 @@ + int sqlite3_stmt_status(sqlite3_stmt *pStmt, int op, int resetFlag){ + Vdbe *pVdbe = (Vdbe*)pStmt; + u32 v; + #ifdef SQLITE_ENABLE_API_ARMOR + if( !pStmt +- || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||op>=ArraySize(pVdbe->aCounter))) ++ || (op!=SQLITE_STMTSTATUS_MEMUSED && (op<0||(op>=ArraySize(pVdbe->aCounter)&&oplookaside.pEnd = db->lookaside.pStart; + sqlite3VdbeDelete(pVdbe); + db->pnBytesFreed = 0; + db->lookaside.pEnd = db->lookaside.pTrueEnd; + sqlite3_mutex_leave(db->mutex); ++ }else if( op>=LIBSQL_STMTSTATUS_BASE ){ ++ v = pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE]; ++ if( resetFlag ) pVdbe->aLibsqlCounter[op - LIBSQL_STMTSTATUS_BASE] = 0; + }else{ + v = pVdbe->aCounter[op]; + if( resetFlag ) pVdbe->aCounter[op] = 0; + } + return (int)v; +diff -u5 -r sqlite-src-3400100-pristine/src/vdbe.c sqlite-src-3400100-modified/src/vdbe.c +--- sqlite-src-3400100-pristine/src/vdbe.c 2022-12-28 06:26:33.000000000 -0800 ++++ sqlite-src-3400100-modified/src/vdbe.c 2023-08-03 14:26:29.473777273 -0700 +@@ -3582,10 +3582,11 @@ + if( pOp->p3 ){ + nEntry = sqlite3BtreeRowCountEst(pCrsr); + }else{ + nEntry = 0; /* Not needed. Only used to silence a warning. */ + rc = sqlite3BtreeCount(db, pCrsr, &nEntry); ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE] += nEntry; + if( rc ) goto abort_due_to_error; + } + pOut = out2Prerelease(p, pOp); + pOut->u.i = nEntry; + goto check_for_interrupt; +@@ -4741,10 +4742,11 @@ + if( eqOnly && r.eqSeen==0 ){ + assert( res!=0 ); + goto seek_not_found; + } + } ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + #ifdef SQLITE_TEST + sqlite3_search_count++; + #endif + if( oc>=OP_SeekGE ){ assert( oc==OP_SeekGE || oc==OP_SeekGT ); + if( res<0 || (res==0 && oc==OP_SeekGT) ){ +@@ -5309,10 +5311,11 @@ + pC->nullRow = 0; + pC->cacheStatus = CACHE_STALE; + pC->deferredMoveto = 0; + VdbeBranchTaken(res!=0,2); + pC->seekResult = res; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + if( res!=0 ){ + assert( rc==SQLITE_OK ); + if( pOp->p2==0 ){ + rc = SQLITE_CORRUPT_BKPT; + }else{ +@@ -5565,10 +5568,11 @@ + } + } + if( pOp->p5 & OPFLAG_ISNOOP ) break; + #endif + ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; + if( pOp->p5 & OPFLAG_LASTROWID ) db->lastRowid = x.nKey; + assert( (pData->flags & (MEM_Blob|MEM_Str))!=0 || pData->n==0 ); + x.pData = pData->z; + x.nData = pData->n; +@@ -5748,10 +5752,11 @@ + pC->seekResult = 0; + if( rc ) goto abort_due_to_error; + + /* Invoke the update-hook if required. */ + if( opflags & OPFLAG_NCHANGE ){ ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + p->nChange++; + if( db->xUpdateCallback && ALWAYS(pTab!=0) && HasRowid(pTab) ){ + db->xUpdateCallback(db->pUpdateArg, SQLITE_DELETE, zDb, pTab->zName, + pC->movetoTarget); + assert( pC->iDb>=0 ); +@@ -6035,10 +6040,11 @@ + rc = sqlite3BtreeLast(pCrsr, &res); + pC->nullRow = (u8)res; + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + if( rc ) goto abort_due_to_error; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + if( pOp->p2>0 ){ + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + } + break; +@@ -6139,10 +6145,11 @@ + pC->deferredMoveto = 0; + pC->cacheStatus = CACHE_STALE; + } + if( rc ) goto abort_due_to_error; + pC->nullRow = (u8)res; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + assert( pOp->p2>0 && pOp->p2nOp ); + VdbeBranchTaken(res!=0,2); + if( res ) goto jump_to_p2; + break; + } +@@ -6243,10 +6250,11 @@ + pC->cacheStatus = CACHE_STALE; + VdbeBranchTaken(rc==SQLITE_OK,2); + if( rc==SQLITE_OK ){ + pC->nullRow = 0; + p->aCounter[pOp->p5]++; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + #ifdef SQLITE_TEST + sqlite3_search_count++; + #endif + goto jump_to_p2_and_check_for_interrupt; + } +@@ -6294,10 +6302,11 @@ + assert( pC!=0 ); + assert( !isSorter(pC) ); + pIn2 = &aMem[pOp->p2]; + assert( (pIn2->flags & MEM_Blob) || (pOp->p5 & OPFLAG_PREFORMAT) ); + if( pOp->p5 & OPFLAG_NCHANGE ) p->nChange++; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + assert( pC->eCurType==CURTYPE_BTREE ); + assert( pC->isTable==0 ); + rc = ExpandBlob(pIn2); + if( rc ) goto abort_due_to_error; + x.nKey = pIn2->n; +@@ -6694,10 +6703,11 @@ + assert( p->readOnly==0 ); + assert( DbMaskTest(p->btreeMask, pOp->p2) ); + rc = sqlite3BtreeClearTable(db->aDb[pOp->p2].pBt, (u32)pOp->p1, &nChange); + if( pOp->p3 ){ + p->nChange += nChange; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE] += nChange; + if( pOp->p3>0 ){ + assert( memIsValid(&aMem[pOp->p3]) ); + memAboutToChange(p, &aMem[pOp->p3]); + aMem[pOp->p3].u.i += nChange; + } +@@ -8176,10 +8186,11 @@ + ** some other method is next invoked on the save virtual table cursor. + */ + rc = pModule->xNext(pCur->uc.pVCur); + sqlite3VtabImportErrmsg(p, pVtab); + if( rc ) goto abort_due_to_error; ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_READ - LIBSQL_STMTSTATUS_BASE]++; + res = pModule->xEof(pCur->uc.pVCur); + VdbeBranchTaken(!res,2); + if( !res ){ + /* If there is data, jump to P2 */ + goto jump_to_p2_and_check_for_interrupt; +@@ -8297,10 +8308,11 @@ + rc = SQLITE_OK; + }else{ + p->errorAction = ((pOp->p5==OE_Replace) ? OE_Abort : pOp->p5); + } + }else{ ++ p->aLibsqlCounter[LIBSQL_STMTSTATUS_ROWS_WRITTEN - LIBSQL_STMTSTATUS_BASE]++; + p->nChange++; + } + if( rc ) goto abort_due_to_error; + } + break; +diff -u5 -r sqlite-src-3400100-pristine/src/vdbeInt.h sqlite-src-3400100-modified/src/vdbeInt.h +--- sqlite-src-3400100-pristine/src/vdbeInt.h 2022-12-28 06:26:33.000000000 -0800 ++++ sqlite-src-3400100-modified/src/vdbeInt.h 2023-08-03 14:26:54.088987824 -0700 +@@ -468,10 +468,11 @@ + bft readOnly:1; /* True for statements that do not write */ + bft bIsReader:1; /* True for statements that read */ + yDbMask btreeMask; /* Bitmask of db->aDb[] entries referenced */ + yDbMask lockMask; /* Subset of btreeMask that requires a lock */ + u32 aCounter[9]; /* Counters used by sqlite3_stmt_status() */ ++ u32 aLibsqlCounter[3]; /* libSQL extension: Counters used by sqlite3_stmt_status()*/ + char *zSql; /* Text of the SQL statement that generated this */ + #ifdef SQLITE_ENABLE_NORMALIZE + char *zNormSql; /* Normalization of the associated SQL statement */ + DblquoteStr *pDblStr; /* List of double-quoted string literals */ + #endif diff --git a/patches/sqlite/README b/patches/sqlite/README new file mode 100644 index 00000000000..23959a004db --- /dev/null +++ b/patches/sqlite/README @@ -0,0 +1,69 @@ +The patches for SQLite are a bit weird due to SQLite's idiosyncratic +build process. + +SQLite comes in two forms. First, there's the "plain" form. It looks a +lot like a typical open-source C project: there's a bunch of .c and .h +files, a Makefile, a configure script, and various other files. + +Second, there's the "amalgamation". This is all of SQLite combined +into two .c and two .h files. workerd consumes this form. + +workerd has to patch SQLite, hence the patches in this directory. +Patching the amalgamation is painful. The vast majority of the code is +in a single 250,000-line .c file. Some pieces of code are duplicated +in the building of the amalgamation, so both spots need patching. +SQLite's tests are not included in the amalgamation, so testing an +amalgamation patch is hard. + +Making workerd consume the plain form would also be painful. To build +the amalgamation, SQLite uses a parser generator, a handful of +utilities written in C, and a whole lot of tclsh scripts all invoked +by a Makefile. To build all this from Bazel, we would need to either +rewrite tens of thousands of lines of TCL or include TCL as a build +dependency of workerd. + +Instead, we have this compromise solution. Each patch in here is +duplicated: one copy for the plain form and one for the amalgamation. +Bazel downloads the SQLite amalgamation and patches it. + +To update to a new SQLite version, obtain the new SQLite in both plain +and amalgamated forms. Apply the plain-form patch to SQLite, fixing as +necessary, and replace the plain-form patch with the fixed version. + +Then, use the patched SQLite to build an amalgamation. Diff that +against the downloaded amalgamation to produce a new amalgamated-form +patch, and replace the existing amalgamated patch with the new one. + + +Example, assuming the new SQLite has been downloaded into the current +directory: + +$ unzip sqlite-src-$VERSION.zip +$ mv sqlite-src-$VERSION sqlite-src-pristine +$ unzip sqlite-src-$VERSION.zip # yes, again +$ mv sqlite-src-$VERSION sqlite-src-modified +$ unzip sqlite-amalgamation-$VERSION.zip +$ mv sqlite-amalgamation-$VERSION.zip sqlite-amalgamation-pristine +$ mkdir sqlite-amalgamation-modified + +Now patch: + +$ cd sqlite-src-modified +$ patch -p1 < /path/to/workerd/patches/sqlite/0001-row-counts-plain.patch +$ ./configure && make test + +Make sure the tests pass. If the patch needed any modification, regenerate it: + +$ diff -u5 -r sqlite-src-pristine sqlite-src-modified \ + | grep -v "Only in sqlite-src-modified" \ + > /path/to/workerd/patches/sqlite/0001-row-counts-plain.patch + +Build the amalagamation and the patch: + +$ make sqlite3.c sqlite3.h sqlite3ext.h shell.c +$ cp shell.c sqlite3.c sqlite3.h sqlite3ext.h ../sqlite-amalgamation-modified/ +$ cd .. +$ diff -u5 -r sqlite-amalgamation-pristine sqlite-amalgamation-modified \ + > /path/to/workerd/patches/sqlite/0001-row-counts-amalgamation.patch + +Repeat for each patch. From 26f0498e303eead759ab4ca9f8767841618eace7 Mon Sep 17 00:00:00 2001 From: Samuel Merritt Date: Tue, 1 Aug 2023 16:21:30 -0700 Subject: [PATCH 2/2] SQL: expose rows read/written as counters. The `storage.sql` object in JS now has two new properties: `rowsRead` and `rowsWritten`, which contain appropriate numbers. Similarly, workerd::api::SqliteDatabase::getRows[Read|Written] provide the same numbers in C++. The counters are only updated after a full result set has been read. I thought about updating the counters as each row was read, but that has the disadvantage of counting rows that weren't useful due to errors. If one wanted to meter usage based on rows, one would only want to count rows for successfully-executed queries. --- src/workerd/api/sql-test.js | 64 ++++++++ src/workerd/api/sql.c++ | 20 +++ src/workerd/api/sql.h | 10 ++ src/workerd/util/sqlite-test.c++ | 252 +++++++++++++++++++++++++++++++ src/workerd/util/sqlite.c++ | 19 +++ src/workerd/util/sqlite.h | 5 + 6 files changed, 370 insertions(+) diff --git a/src/workerd/api/sql-test.js b/src/workerd/api/sql-test.js index f2db9a3fba1..c112e319c81 100644 --- a/src/workerd/api/sql-test.js +++ b/src/workerd/api/sql-test.js @@ -670,6 +670,64 @@ async function test(storage) { ) } +async function testIoStats(storage) { + const sql = storage.sql; + + sql.exec(`CREATE TABLE tbl (id INTEGER PRIMARY KEY, value TEXT)`); + sql.exec(`INSERT INTO tbl (id, value) VALUES (?, ?)`, 100000, "arbitrary-initial-value"); + await scheduler.wait(1); + + // When writing, the rowsWritten count goes up. + { + const cursor = sql.exec(`INSERT INTO tbl (id, value) VALUES (?, ?)`, 1, "arbitrary-value"); + Array.from(cursor); // Consume all the results + assert.equal(cursor.rowsWritten, 1); + } + + // When reading, the rowsRead count goes up. + { + const cursor = sql.exec(`SELECT * FROM tbl`); + Array.from(cursor); // Consume all the results + assert.equal(cursor.rowsRead, 2); + } + + // Each invocation of a prepared statement gets its own counters. + { + const id1 = 101; + const id2 = 202; + + const prepared = sql.prepare(`INSERT INTO tbl (id, value) VALUES (?, ?)`); + const cursor123 = prepared(id1, "value1"); + Array.from(cursor123); + assert.equal(cursor123.rowsWritten, 1); + + const cursor456 = prepared(id2, "value2"); + Array.from(cursor456); + assert.equal(cursor456.rowsWritten, 1); + assert.equal(cursor123.rowsWritten, 1); // remained unchanged + } + + // Row counters are updated as you consume the cursor. + { + sql.exec(`DELETE FROM tbl`); + const prepared = sql.prepare(`INSERT INTO tbl (id, value) VALUES (?, ?)`); + for (let i = 1; i <= 10; i++) { + Array.from(prepared(i, "value" + i)); + } + + const cursor = sql.exec(`SELECT * FROM tbl`); + const resultsIterator = cursor[Symbol.iterator](); + let rowsSeen = 0; + while (true) { + const result = resultsIterator.next(); + if (result.done) { + break; + } + assert.equal(++rowsSeen, cursor.rowsRead); + } + } +} + async function testForeignKeys(storage) { const sql = storage.sql @@ -763,6 +821,9 @@ export class DurableObjectExample { // abort() always throws. throw new Error("can't get here") + } else if (req.url.endsWith('/sql-test-io-stats')) { + await testIoStats(this.state.storage) + return Response.json({ ok: true }) } throw new Error('unknown url: ' + req.url) @@ -783,6 +844,9 @@ export default { // Test SQL API assert.deepEqual(await doReq('sql-test'), { ok: true }) + // Test SQL IO stats + assert.deepEqual(await doReq("sql-test-io-stats"), {ok: true}); + // Test defer_foreign_keys (explodes the DO) await assert.rejects(async () => { await doReq('sql-test-foreign-keys') diff --git a/src/workerd/api/sql.c++ b/src/workerd/api/sql.c++ index adbdb97fa30..db2659bdae9 100644 --- a/src/workerd/api/sql.c++ +++ b/src/workerd/api/sql.c++ @@ -91,6 +91,22 @@ void SqlStorage::Cursor::CachedColumnNames::ensureInitialized( } } +double SqlStorage::Cursor::getRowsRead() { + KJ_IF_MAYBE(st, state) { + return static_cast((**st).query.getRowsRead()); + } else { + return static_cast(rowsRead); + } +} + +double SqlStorage::Cursor::getRowsWritten() { + KJ_IF_MAYBE(st, state) { + return static_cast((**st).query.getRowsWritten()); + } else { + return static_cast(rowsWritten); + } +} + jsg::Ref SqlStorage::Cursor::rows(jsg::Lock& js) { KJ_IF_MAYBE(s, state) { cachedColumnNames.ensureInitialized(js, (*s)->query); @@ -168,7 +184,11 @@ auto SqlStorage::Cursor::iteratorImpl(jsg::Lock& js, jsg::Ref& obj, Func } auto& query = state.query; + if (query.isDone()) { + // Save off row counts before the query goes away. + obj->rowsRead = query.getRowsRead(); + obj->rowsWritten = query.getRowsWritten(); // Clean up the query proactively. obj->state = nullptr; return nullptr; diff --git a/src/workerd/api/sql.h b/src/workerd/api/sql.h index 3044e651fb1..4cce429e595 100644 --- a/src/workerd/api/sql.h +++ b/src/workerd/api/sql.h @@ -97,11 +97,16 @@ class SqlStorage::Cursor final: public jsg::Object { cachedColumnNames(cachedColumnNames) {} ~Cursor() noexcept(false); + double getRowsRead(); + double getRowsWritten(); + kj::Array> getColumnNames(jsg::Lock& js); JSG_RESOURCE_TYPE(Cursor, CompatibilityFlags::Reader flags) { JSG_ITERABLE(rows); JSG_METHOD(raw); JSG_READONLY_PROTOTYPE_PROPERTY(columnNames, getColumnNames); + JSG_READONLY_PROTOTYPE_PROPERTY(rowsRead, getRowsRead); + JSG_READONLY_PROTOTYPE_PROPERTY(rowsWritten, getRowsWritten); } @@ -164,6 +169,11 @@ class SqlStorage::Cursor final: public jsg::Object { kj::Maybe> statement; // If this cursor was created from a prepared statement, this keeps the statement object alive. + uint64_t rowsRead = 0; + uint64_t rowsWritten = 0; + // Row IO counts. These are updated as the query runs. We keep these outside the State so they + // remain available even after the query is done or canceled. + kj::Maybe ownCachedColumnNames; CachedColumnNames& cachedColumnNames; diff --git a/src/workerd/util/sqlite-test.c++ b/src/workerd/util/sqlite-test.c++ index 9ea2b6787bc..1a9709761ae 100644 --- a/src/workerd/util/sqlite-test.c++ +++ b/src/workerd/util/sqlite-test.c++ @@ -3,6 +3,7 @@ // https://opensource.org/licenses/Apache-2.0 #include "sqlite.h" +#include #include #include #include @@ -366,5 +367,256 @@ KJ_TEST("SQLite onWrite callback") { KJ_EXPECT(sawWrite); } +struct RowCounts { + uint64_t found; + uint64_t read; + uint64_t written; +}; + +template +RowCounts countRowsTouched(SqliteDatabase& db, SqliteDatabase::Regulator& regulator, kj::StringPtr sqlCode, Params... bindParams) { + uint64_t rowsFound = 0; + + // Runs a query; retrieves and discards all the data. + auto query = db.run(regulator, sqlCode, bindParams...); + while (!query.isDone()) { + rowsFound++; + query.nextRow(); + } + + return {.found = rowsFound, + .read = query.getRowsRead(), + .written = query.getRowsWritten()}; +} + +template +RowCounts countRowsTouched(SqliteDatabase& db, kj::StringPtr sqlCode, Params... bindParams) { + return countRowsTouched(db, SqliteDatabase::TRUSTED, sqlCode, std::forward(bindParams)...); +} + +KJ_TEST("SQLite read row counters (basic)") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + db.run(R"( + CREATE TABLE things ( + id INTEGER PRIMARY KEY, + unindexed_int INTEGER, + value TEXT + ); + )"); + + constexpr int dbRowCount = 1000; + auto insertStmt = db.prepare("INSERT INTO things (id, unindexed_int, value) VALUES (?, ?, ?)"); + for (int i = 0; i < dbRowCount; i++) { + insertStmt.run(i, i * 1000, kj::str("value", i)); + } + + // Sanity check that the inserts worked. + { + auto getCount = db.prepare("SELECT COUNT(*) FROM things"); + KJ_EXPECT(getCount.run().getInt(0) == dbRowCount); + } + + // Selecting all the rows reads all the rows. + { + RowCounts stats = countRowsTouched(db, "SELECT * FROM things"); + KJ_EXPECT(stats.found == dbRowCount); + KJ_EXPECT(stats.read == dbRowCount); + KJ_EXPECT(stats.written == 0); + } + + // Selecting one row using an index reads one row. + { + RowCounts stats = countRowsTouched(db, "SELECT * FROM things WHERE id=?", 5); + KJ_EXPECT(stats.found == 1); + KJ_EXPECT(stats.read == 1); + KJ_EXPECT(stats.written == 0); + } + + // Selecting one row using an reads one row, even if that row is in the middle of the table. + { + RowCounts stats = countRowsTouched(db, "SELECT * FROM things WHERE id=?", dbRowCount / 2); + KJ_EXPECT(stats.found == 1); + KJ_EXPECT(stats.read == 1); + KJ_EXPECT(stats.written == 0); + } + + // Selecting a row by an unindexed value reads the whole table. + { + RowCounts stats = countRowsTouched(db, "SELECT * FROM things WHERE unindexed_int = ?", 5000); + KJ_EXPECT(stats.found == 1); + KJ_EXPECT(stats.read == dbRowCount); + KJ_EXPECT(stats.written == 0); + } + + // Selecting an unindexed aggregate scans all the rows, which counts as reading them. + { + RowCounts stats = countRowsTouched(db, "SELECT MAX(unindexed_int) FROM things"); + KJ_EXPECT(stats.found == 1); + KJ_EXPECT(stats.read == dbRowCount); + KJ_EXPECT(stats.written == 0); + } + + // Selecting an indexed aggregate can use the index, so it only reads the row it found. + { + RowCounts stats = countRowsTouched(db, "SELECT MIN(id) FROM things"); + KJ_EXPECT(stats.found == 1); + KJ_EXPECT(stats.read == 1); + KJ_EXPECT(stats.written == 0); + } + + // Selecting with a limit only reads the returned rows. + { + RowCounts stats = countRowsTouched(db, "SELECT * FROM things LIMIT 5"); + KJ_EXPECT(stats.found == 5); + KJ_EXPECT(stats.read == 5); + KJ_EXPECT(stats.written == 0); + } +} + +KJ_TEST("SQLite write row counters (basic)") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + db.run(R"( + CREATE TABLE things ( + id INTEGER PRIMARY KEY + ); + )"); + + db.run(R"( + CREATE TABLE unindexed_things ( + id INTEGER + ); + )"); + + // Inserting a row counts as one row written. + { + RowCounts stats = countRowsTouched(db, "INSERT INTO unindexed_things (id) VALUES (?)", 1); + KJ_EXPECT(stats.read == 0); + KJ_EXPECT(stats.written == 1); + } + + // Inserting a row into a table with a primary key will also do a read (to ensure there's no + // duplicate PK). + { + RowCounts stats = countRowsTouched(db, "INSERT INTO things (id) VALUES (?)", 1); + KJ_EXPECT(stats.read == 1); + KJ_EXPECT(stats.written == 1); + } + + // Deleting a row counts as a write. + { + RowCounts stats = countRowsTouched(db, "INSERT INTO things (id) VALUES (?)", 123); + KJ_EXPECT(stats.written == 1); + + stats = countRowsTouched(db, "DELETE FROM things WHERE id=?", 123); + KJ_EXPECT(stats.read == 1); + KJ_EXPECT(stats.written == 1); + } + + // Deleting nothing is not a write. + { + RowCounts stats = countRowsTouched(db, "DELETE FROM things WHERE id=?", 998877112233); + KJ_EXPECT(stats.written == 0); + } + + // Inserting many things is many writes. + { + db.run("DELETE FROM things"); + db.run("INSERT INTO things (id) VALUES (1)"); + db.run("INSERT INTO things (id) VALUES (3)"); + db.run("INSERT INTO things (id) VALUES (5)"); + + RowCounts stats = countRowsTouched(db, "INSERT INTO unindexed_things (id) SELECT id FROM things"); + KJ_EXPECT(stats.read == 3); + KJ_EXPECT(stats.written == 3); + } + + // Each updated row is a write. + { + db.run("DELETE FROM unindexed_things"); + db.run("INSERT INTO unindexed_things (id) VALUES (1)"); + db.run("INSERT INTO unindexed_things (id) VALUES (2)"); + db.run("INSERT INTO unindexed_things (id) VALUES (3)"); + db.run("INSERT INTO unindexed_things (id) VALUES (4)"); + + RowCounts stats = countRowsTouched(db, "UPDATE unindexed_things SET id = id * 10 WHERE id >= 3"); + KJ_EXPECT(stats.written == 2); + } + + // On an indexed table, each updated row is two writes. This is probably due to the index update. + { + db.run("DELETE FROM things"); + db.run("INSERT INTO things (id) VALUES (1)"); + db.run("INSERT INTO things (id) VALUES (2)"); + db.run("INSERT INTO things (id) VALUES (3)"); + db.run("INSERT INTO things (id) VALUES (4)"); + + RowCounts stats = countRowsTouched(db, "UPDATE things SET id = id * 10 WHERE id >= 3"); + KJ_EXPECT(stats.read >= 4); // At least one read per updated row + KJ_EXPECT(stats.written == 4); + } +} + +KJ_TEST("SQLite row counters with triggers") { + auto dir = kj::newInMemoryDirectory(kj::nullClock()); + SqliteDatabase::Vfs vfs(*dir); + SqliteDatabase db(vfs, kj::Path({"foo"}), kj::WriteMode::CREATE | kj::WriteMode::MODIFY); + + class RegulatorImpl: public SqliteDatabase::Regulator { + public: + RegulatorImpl() = default; + + bool isAllowedTrigger(kj::StringPtr name) override { + // SqliteDatabase::TRUSTED doesn't let us use triggers at all. + return true; + } + }; + + RegulatorImpl regulator; + + db.run(R"( + CREATE TABLE things ( + id INTEGER PRIMARY KEY + ); + + CREATE TABLE log ( + id INTEGER, + verb TEXT + ); + + CREATE TRIGGER log_inserts AFTER INSERT ON things + BEGIN + insert into log (id, verb) VALUES (NEW.id, "INSERT"); + END; + + CREATE TRIGGER log_deletes AFTER DELETE ON things + BEGIN + insert into log (id, verb) VALUES (OLD.id, "DELETE"); + END; + )"); + + // Each insert incurs two writes: one for the row in `things` and one for the row in `log`. + { + RowCounts stats = countRowsTouched(db, regulator, "INSERT INTO things (id) VALUES (1)"); + KJ_EXPECT(stats.written == 2); + } + + // A deletion incurs two writes: one for the row and one for the log. + { + db.run(regulator, "DELETE FROM things"); + db.run(regulator, "INSERT INTO things (id) VALUES (1)"); + db.run(regulator, "INSERT INTO things (id) VALUES (2)"); + db.run(regulator, "INSERT INTO things (id) VALUES (3)"); + + RowCounts stats = countRowsTouched(db, regulator, "DELETE FROM things"); + KJ_EXPECT(stats.written == 6); + } +} + } // namespace } // namespace workerd diff --git a/src/workerd/util/sqlite.c++ b/src/workerd/util/sqlite.c++ index 8a1ba88faab..6e134367b84 100644 --- a/src/workerd/util/sqlite.c++ +++ b/src/workerd/util/sqlite.c++ @@ -760,6 +760,9 @@ SqliteDatabase::Statement SqliteDatabase::prepare(Regulator& regulator, kj::Stri SqliteDatabase::Query::Query(SqliteDatabase& db, Regulator& regulator, Statement& statement, kj::ArrayPtr bindings) : db(db), regulator(regulator), statement(statement) { + // If the statement was used for a previous query, then its row counters contain data from that + // query's execution. Reset them to zero. + resetRowCounters(); init(bindings); } @@ -828,6 +831,22 @@ void SqliteDatabase::Query::bind(uint i, ValuePtr value) { } } +uint64_t SqliteDatabase::Query::getRowsRead() { + KJ_REQUIRE(statement != nullptr); + return sqlite3_stmt_status(statement, LIBSQL_STMTSTATUS_ROWS_READ, 0); +} + +uint64_t SqliteDatabase::Query::getRowsWritten() { + KJ_REQUIRE(statement != nullptr); + return sqlite3_stmt_status(statement, LIBSQL_STMTSTATUS_ROWS_WRITTEN, 0); +} + +void SqliteDatabase::Query::resetRowCounters() { + KJ_REQUIRE(statement != nullptr); + sqlite3_stmt_status(statement, LIBSQL_STMTSTATUS_ROWS_READ, 1); + sqlite3_stmt_status(statement, LIBSQL_STMTSTATUS_ROWS_WRITTEN, 1); +} + void SqliteDatabase::Query::bind(uint i, kj::ArrayPtr value) { SQLITE_CALL(sqlite3_bind_blob(statement, i+1, value.begin(), value.size(), SQLITE_STATIC)); } diff --git a/src/workerd/util/sqlite.h b/src/workerd/util/sqlite.h index 58166bbd16f..ca7ca52a0d3 100644 --- a/src/workerd/util/sqlite.h +++ b/src/workerd/util/sqlite.h @@ -217,6 +217,10 @@ class SqliteDatabase::Query { ~Query() noexcept(false); KJ_DISALLOW_COPY_AND_MOVE(Query); + uint64_t getRowsRead(); + uint64_t getRowsWritten(); + // Row IO counters. + bool isDone() { return done; } // If true, there are no more rows. (When true, the methods below must not be called.) @@ -292,6 +296,7 @@ class SqliteDatabase::Query { void checkRequirements(size_t size); void init(kj::ArrayPtr bindings); + void resetRowCounters(); void bind(uint column, ValuePtr value); void bind(uint column, kj::ArrayPtr value);