diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 35a0110..bbd0217 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -61,10 +61,13 @@ jobs: -e MINIO_ROOT_USER=root \ -e MINIO_ROOT_PASSWORD=password \ -e MINIO_DOMAIN=minio \ - -v ${{ github.workspace }}/assets/tpch:/input minio/minio:latest server /data/ + -v ${{ github.workspace }}/assets/tpch:/input \ + -v ${{ github.workspace }}/assets/incremental:/input_incremental \ + minio/minio:latest server /data/ until (docker exec minio mc alias set minio http://localhost:9000 root password) do echo '... waiting ...' && sleep 1; done; docker exec minio mc mb minio/warehouse docker exec minio mc cp -r /input/tpch.sf01/ minio/warehouse/tpch.sf01/ + docker exec minio mc cp -r /input_incremental/ minio/warehouse/incremental/; docker run -d --name rest \ --network=iceberg_net \ -p 8181:8181 \ diff --git a/.gitignore b/.gitignore index 186df43..59fd88d 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ **/.env iceberg_rust_ffi/target +iceberg_rust_ffi/target/**/* iceberg_rust_ffi/integration_test **/*.dylib -**/.claude \ No newline at end of file +**/.claude +**/.DS_Store \ No newline at end of file diff --git a/CLAUDE.md b/CLAUDE.md index dd3b5ae..2e6b266 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -34,11 +34,6 @@ This project provides a **Foreign Function Interface (FFI)** for Apache Iceberg, - **Julia Integration**: Conditional compilation features for Julia interop (`julia` feature flag) - **Memory Management**: Safe FFI patterns with proper cleanup functions -#### C header (`include/iceberg_rust_ffi.h`) -- **Manual Generation**: C header is not generated right now. Whenever you make a change in the Rust library, examine whether the header should be updated. -- **C99 Compatible**: Ensures compatibility with standard C compilers -- **Response Structures**: Async operations return response structures with context for cancellation - ### FFI Design Patterns #### Async Operations with Callbacks diff --git a/Makefile b/Makefile index 42b1364..1ca6832 100644 --- a/Makefile +++ b/Makefile @@ -48,7 +48,7 @@ test: build exit 1; \ fi @set -a && . ./.env && set +a && \ - export ICEBERG_RUST_LIB=$(TARGET_DIR) && \ + export ICEBERG_RUST_LIB=$$(pwd)/$(TARGET_DIR) && \ $(JULIA_THREADS_ENV) julia --project=. -e 'using Pkg; Pkg.test()' # Start Julia REPL with environment configured (requires .env file) @@ -58,7 +58,7 @@ repl: build exit 1; \ fi @set -a && . ./.env && set +a && \ - export ICEBERG_RUST_LIB=$(TARGET_DIR) && \ + export ICEBERG_RUST_LIB=$$(pwd)/$(TARGET_DIR) && \ $(JULIA_THREADS_ENV) julia --project=. # Clean build artifacts diff --git a/assets/incremental/test1/data/00000-1-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet b/assets/incremental/test1/data/00000-1-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet new file mode 100644 index 0000000..9e6b467 Binary files /dev/null and b/assets/incremental/test1/data/00000-1-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00000-16-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet b/assets/incremental/test1/data/00000-16-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet new file mode 100644 index 0000000..9e6b467 Binary files /dev/null and b/assets/incremental/test1/data/00000-16-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00000-21-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet b/assets/incremental/test1/data/00000-21-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet new file mode 100644 index 0000000..9e6b467 Binary files /dev/null and b/assets/incremental/test1/data/00000-21-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00000-31-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet b/assets/incremental/test1/data/00000-31-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet new file mode 100644 index 0000000..9e6b467 Binary files /dev/null and b/assets/incremental/test1/data/00000-31-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00000-38-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet b/assets/incremental/test1/data/00000-38-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet new file mode 100644 index 0000000..e11cdc4 Binary files /dev/null and b/assets/incremental/test1/data/00000-38-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00000-43-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet b/assets/incremental/test1/data/00000-43-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet new file mode 100644 index 0000000..25adefb Binary files /dev/null and b/assets/incremental/test1/data/00000-43-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00000-49-3e01c1a1-4b17-4791-851a-ef2f47d9e688-00001-deletes.parquet b/assets/incremental/test1/data/00000-49-3e01c1a1-4b17-4791-851a-ef2f47d9e688-00001-deletes.parquet new file mode 100644 index 0000000..1bb1329 Binary files /dev/null and b/assets/incremental/test1/data/00000-49-3e01c1a1-4b17-4791-851a-ef2f47d9e688-00001-deletes.parquet differ diff --git a/assets/incremental/test1/data/00000-49-3e01c1a1-4b17-4791-851a-ef2f47d9e688-00002-deletes.parquet b/assets/incremental/test1/data/00000-49-3e01c1a1-4b17-4791-851a-ef2f47d9e688-00002-deletes.parquet new file mode 100644 index 0000000..3c499cd Binary files /dev/null and b/assets/incremental/test1/data/00000-49-3e01c1a1-4b17-4791-851a-ef2f47d9e688-00002-deletes.parquet differ diff --git a/assets/incremental/test1/data/00001-17-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet b/assets/incremental/test1/data/00001-17-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet new file mode 100644 index 0000000..2696e94 Binary files /dev/null and b/assets/incremental/test1/data/00001-17-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00001-2-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet b/assets/incremental/test1/data/00001-2-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet new file mode 100644 index 0000000..2696e94 Binary files /dev/null and b/assets/incremental/test1/data/00001-2-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00001-22-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet b/assets/incremental/test1/data/00001-22-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet new file mode 100644 index 0000000..2696e94 Binary files /dev/null and b/assets/incremental/test1/data/00001-22-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00001-32-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet b/assets/incremental/test1/data/00001-32-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet new file mode 100644 index 0000000..2696e94 Binary files /dev/null and b/assets/incremental/test1/data/00001-32-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00001-39-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet b/assets/incremental/test1/data/00001-39-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet new file mode 100644 index 0000000..3ed25e6 Binary files /dev/null and b/assets/incremental/test1/data/00001-39-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00001-44-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet b/assets/incremental/test1/data/00001-44-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet new file mode 100644 index 0000000..22540ab Binary files /dev/null and b/assets/incremental/test1/data/00001-44-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00002-18-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet b/assets/incremental/test1/data/00002-18-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet new file mode 100644 index 0000000..87568a1 Binary files /dev/null and b/assets/incremental/test1/data/00002-18-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00002-23-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet b/assets/incremental/test1/data/00002-23-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet new file mode 100644 index 0000000..87568a1 Binary files /dev/null and b/assets/incremental/test1/data/00002-23-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00002-3-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet b/assets/incremental/test1/data/00002-3-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet new file mode 100644 index 0000000..87568a1 Binary files /dev/null and b/assets/incremental/test1/data/00002-3-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00002-33-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet b/assets/incremental/test1/data/00002-33-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet new file mode 100644 index 0000000..87568a1 Binary files /dev/null and b/assets/incremental/test1/data/00002-33-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00002-40-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet b/assets/incremental/test1/data/00002-40-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet new file mode 100644 index 0000000..64683bc Binary files /dev/null and b/assets/incremental/test1/data/00002-40-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00002-45-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet b/assets/incremental/test1/data/00002-45-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet new file mode 100644 index 0000000..64c0cba Binary files /dev/null and b/assets/incremental/test1/data/00002-45-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00003-19-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet b/assets/incremental/test1/data/00003-19-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet new file mode 100644 index 0000000..998f61f Binary files /dev/null and b/assets/incremental/test1/data/00003-19-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00003-24-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet b/assets/incremental/test1/data/00003-24-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet new file mode 100644 index 0000000..998f61f Binary files /dev/null and b/assets/incremental/test1/data/00003-24-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00003-34-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet b/assets/incremental/test1/data/00003-34-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet new file mode 100644 index 0000000..998f61f Binary files /dev/null and b/assets/incremental/test1/data/00003-34-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00003-4-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet b/assets/incremental/test1/data/00003-4-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet new file mode 100644 index 0000000..998f61f Binary files /dev/null and b/assets/incremental/test1/data/00003-4-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00003-41-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet b/assets/incremental/test1/data/00003-41-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet new file mode 100644 index 0000000..a8a92cf Binary files /dev/null and b/assets/incremental/test1/data/00003-41-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00003-46-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet b/assets/incremental/test1/data/00003-46-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet new file mode 100644 index 0000000..1b02ec2 Binary files /dev/null and b/assets/incremental/test1/data/00003-46-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00004-20-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet b/assets/incremental/test1/data/00004-20-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet new file mode 100644 index 0000000..e09ff91 Binary files /dev/null and b/assets/incremental/test1/data/00004-20-e1fe0411-f9e9-436a-aa67-57e182f7b728-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00004-25-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet b/assets/incremental/test1/data/00004-25-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet new file mode 100644 index 0000000..e09ff91 Binary files /dev/null and b/assets/incremental/test1/data/00004-25-a87e1828-267e-432e-8980-dc0dcb1b5cd1-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00004-35-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet b/assets/incremental/test1/data/00004-35-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet new file mode 100644 index 0000000..e09ff91 Binary files /dev/null and b/assets/incremental/test1/data/00004-35-b64e9ce4-f37b-4fdc-8534-b2664d773b07-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00004-42-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet b/assets/incremental/test1/data/00004-42-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet new file mode 100644 index 0000000..71c9e78 Binary files /dev/null and b/assets/incremental/test1/data/00004-42-a951d037-3439-447f-8ab4-7f8c40f42daa-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00004-47-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet b/assets/incremental/test1/data/00004-47-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet new file mode 100644 index 0000000..d0d873c Binary files /dev/null and b/assets/incremental/test1/data/00004-47-4aa600ad-8ce1-45f4-b23a-dd3b4adc83ed-0-00001.parquet differ diff --git a/assets/incremental/test1/data/00004-5-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet b/assets/incremental/test1/data/00004-5-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet new file mode 100644 index 0000000..e09ff91 Binary files /dev/null and b/assets/incremental/test1/data/00004-5-1215e431-f3d7-48ba-a5de-66ab60e3010a-0-00001.parquet differ diff --git a/assets/incremental/test1/metadata/00000-9b1f6e90-9146-4446-bc93-9625ad2f12de.metadata.json b/assets/incremental/test1/metadata/00000-9b1f6e90-9146-4446-bc93-9625ad2f12de.metadata.json new file mode 100644 index 0000000..fdccfa0 --- /dev/null +++ b/assets/incremental/test1/metadata/00000-9b1f6e90-9146-4446-bc93-9625ad2f12de.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"bc0f34cc-7492-4fb0-bd26-7e6749d1aed3","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761916004464,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:06:44.221713590Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":2201372158267155259,"refs":{"main":{"snapshot-id":2201372158267155259,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":2201372158267155259,"timestamp-ms":1761916004464,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-2201372158267155259-1-565b26e5-f200-448f-8ccf-02cebeb64f15.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916004464,"snapshot-id":2201372158267155259}],"metadata-log":[]} \ No newline at end of file diff --git a/assets/incremental/test1/metadata/00000-cf659803-d5fa-44fe-9c86-ecfc225f11ce.metadata.json b/assets/incremental/test1/metadata/00000-cf659803-d5fa-44fe-9c86-ecfc225f11ce.metadata.json new file mode 100644 index 0000000..25fe5ab --- /dev/null +++ b/assets/incremental/test1/metadata/00000-cf659803-d5fa-44fe-9c86-ecfc225f11ce.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"8d69da49-0398-4107-8ede-956e3c1031ed","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761915547394,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T12:59:06.941439875Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":7624749623479285340,"refs":{"main":{"snapshot-id":7624749623479285340,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":7624749623479285340,"timestamp-ms":1761915547394,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-7624749623479285340-1-06af070a-6938-43ea-94f6-659a8b7b6799.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761915547394,"snapshot-id":7624749623479285340}],"metadata-log":[]} \ No newline at end of file diff --git a/assets/incremental/test1/metadata/00000-d4d80e79-dc1a-4a50-a636-25cf30946d09.metadata.json b/assets/incremental/test1/metadata/00000-d4d80e79-dc1a-4a50-a636-25cf30946d09.metadata.json new file mode 100644 index 0000000..8337653 --- /dev/null +++ b/assets/incremental/test1/metadata/00000-d4d80e79-dc1a-4a50-a636-25cf30946d09.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"208df1e5-aa98-49e4-91f8-0fff75d991ed","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761915998275,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:06:38.150855545Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":935682165401343509,"refs":{"main":{"snapshot-id":935682165401343509,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":935682165401343509,"timestamp-ms":1761915998275,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-935682165401343509-1-94811a1f-2293-4ebd-a227-cf4fc471fe85.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761915998275,"snapshot-id":935682165401343509}],"metadata-log":[]} \ No newline at end of file diff --git a/assets/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json b/assets/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json new file mode 100644 index 0000000..8b93fc0 --- /dev/null +++ b/assets/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761916064879,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":6544470421042678943,"refs":{"main":{"snapshot-id":6544470421042678943,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943}],"metadata-log":[]} \ No newline at end of file diff --git a/assets/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json b/assets/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json new file mode 100644 index 0000000..4bf4e92 --- /dev/null +++ b/assets/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":2,"last-updated-ms":1761916192089,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":6540713100348352610,"refs":{"main":{"snapshot-id":6540713100348352610,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0},{"sequence-number":2,"snapshot-id":6540713100348352610,"parent-snapshot-id":6544470421042678943,"timestamp-ms":1761916192089,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2433","changed-partition-count":"1","total-records":"109","total-files-size":"4637","total-data-files":"10","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943},{"timestamp-ms":1761916192089,"snapshot-id":6540713100348352610}],"metadata-log":[{"timestamp-ms":1761916064879,"metadata-file":"s3://warehouse/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json"}]} \ No newline at end of file diff --git a/assets/incremental/test1/metadata/00002-ec6c7e37-0b8f-44b3-bb11-7fee9a9091ca.metadata.json b/assets/incremental/test1/metadata/00002-ec6c7e37-0b8f-44b3-bb11-7fee9a9091ca.metadata.json new file mode 100644 index 0000000..7ff4ba3 --- /dev/null +++ b/assets/incremental/test1/metadata/00002-ec6c7e37-0b8f-44b3-bb11-7fee9a9091ca.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":3,"last-updated-ms":1761916215684,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":8895755745308925611,"refs":{"main":{"snapshot-id":8895755745308925611,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0},{"sequence-number":2,"snapshot-id":6540713100348352610,"parent-snapshot-id":6544470421042678943,"timestamp-ms":1761916192089,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2433","changed-partition-count":"1","total-records":"109","total-files-size":"4637","total-data-files":"10","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro","schema-id":0},{"sequence-number":3,"snapshot-id":8895755745308925611,"parent-snapshot-id":6540713100348352610,"timestamp-ms":1761916215684,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2490","changed-partition-count":"1","total-records":"208","total-files-size":"7127","total-data-files":"15","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-8895755745308925611-1-2e224b4d-1419-4378-b5ae-76f3ee5e039d.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943},{"timestamp-ms":1761916192089,"snapshot-id":6540713100348352610},{"timestamp-ms":1761916215684,"snapshot-id":8895755745308925611}],"metadata-log":[{"timestamp-ms":1761916064879,"metadata-file":"s3://warehouse/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json"},{"timestamp-ms":1761916192089,"metadata-file":"s3://warehouse/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json"}]} \ No newline at end of file diff --git a/assets/incremental/test1/metadata/00003-359e8bb8-1e5d-46d2-bcde-fdaeaa41114f.metadata.json b/assets/incremental/test1/metadata/00003-359e8bb8-1e5d-46d2-bcde-fdaeaa41114f.metadata.json new file mode 100644 index 0000000..ff6b3d9 --- /dev/null +++ b/assets/incremental/test1/metadata/00003-359e8bb8-1e5d-46d2-bcde-fdaeaa41114f.metadata.json @@ -0,0 +1 @@ +{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":4,"last-updated-ms":1761916266468,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":6832180054960511692,"refs":{"main":{"snapshot-id":6832180054960511692,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0},{"sequence-number":2,"snapshot-id":6540713100348352610,"parent-snapshot-id":6544470421042678943,"timestamp-ms":1761916192089,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2433","changed-partition-count":"1","total-records":"109","total-files-size":"4637","total-data-files":"10","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro","schema-id":0},{"sequence-number":3,"snapshot-id":8895755745308925611,"parent-snapshot-id":6540713100348352610,"timestamp-ms":1761916215684,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2490","changed-partition-count":"1","total-records":"208","total-files-size":"7127","total-data-files":"15","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-8895755745308925611-1-2e224b4d-1419-4378-b5ae-76f3ee5e039d.avro","schema-id":0},{"sequence-number":4,"snapshot-id":6832180054960511692,"parent-snapshot-id":8895755745308925611,"timestamp-ms":1761916266468,"summary":{"operation":"delete","spark.app.id":"local-1761914226119","added-position-delete-files":"2","added-delete-files":"2","added-files-size":"2924","added-position-deletes":"2","changed-partition-count":"1","total-records":"208","total-files-size":"10051","total-data-files":"15","total-delete-files":"2","total-position-deletes":"2","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6832180054960511692-1-f6b39896-f4f2-43d8-8f02-990884cd7287.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943},{"timestamp-ms":1761916192089,"snapshot-id":6540713100348352610},{"timestamp-ms":1761916215684,"snapshot-id":8895755745308925611},{"timestamp-ms":1761916266468,"snapshot-id":6832180054960511692}],"metadata-log":[{"timestamp-ms":1761916064879,"metadata-file":"s3://warehouse/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json"},{"timestamp-ms":1761916192089,"metadata-file":"s3://warehouse/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json"},{"timestamp-ms":1761916215684,"metadata-file":"s3://warehouse/incremental/test1/metadata/00002-ec6c7e37-0b8f-44b3-bb11-7fee9a9091ca.metadata.json"}]} \ No newline at end of file diff --git a/assets/incremental/test1/metadata/13d31e71-0067-4bf5-92a2-2bb1b51cc181-m0.avro b/assets/incremental/test1/metadata/13d31e71-0067-4bf5-92a2-2bb1b51cc181-m0.avro new file mode 100644 index 0000000..c3d52ab Binary files /dev/null and b/assets/incremental/test1/metadata/13d31e71-0067-4bf5-92a2-2bb1b51cc181-m0.avro differ diff --git a/assets/incremental/test1/metadata/154380bf-11a5-4ac4-8c8f-6ade7758b93c-m0.avro b/assets/incremental/test1/metadata/154380bf-11a5-4ac4-8c8f-6ade7758b93c-m0.avro new file mode 100644 index 0000000..4f51c83 Binary files /dev/null and b/assets/incremental/test1/metadata/154380bf-11a5-4ac4-8c8f-6ade7758b93c-m0.avro differ diff --git a/assets/incremental/test1/metadata/2e224b4d-1419-4378-b5ae-76f3ee5e039d-m0.avro b/assets/incremental/test1/metadata/2e224b4d-1419-4378-b5ae-76f3ee5e039d-m0.avro new file mode 100644 index 0000000..6399932 Binary files /dev/null and b/assets/incremental/test1/metadata/2e224b4d-1419-4378-b5ae-76f3ee5e039d-m0.avro differ diff --git a/assets/incremental/test1/metadata/f6b39896-f4f2-43d8-8f02-990884cd7287-m0.avro b/assets/incremental/test1/metadata/f6b39896-f4f2-43d8-8f02-990884cd7287-m0.avro new file mode 100644 index 0000000..20fac55 Binary files /dev/null and b/assets/incremental/test1/metadata/f6b39896-f4f2-43d8-8f02-990884cd7287-m0.avro differ diff --git a/assets/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro b/assets/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro new file mode 100644 index 0000000..9c8061a Binary files /dev/null and b/assets/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro differ diff --git a/assets/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro b/assets/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro new file mode 100644 index 0000000..504a7cb Binary files /dev/null and b/assets/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro differ diff --git a/assets/incremental/test1/metadata/snap-6832180054960511692-1-f6b39896-f4f2-43d8-8f02-990884cd7287.avro b/assets/incremental/test1/metadata/snap-6832180054960511692-1-f6b39896-f4f2-43d8-8f02-990884cd7287.avro new file mode 100644 index 0000000..464222c Binary files /dev/null and b/assets/incremental/test1/metadata/snap-6832180054960511692-1-f6b39896-f4f2-43d8-8f02-990884cd7287.avro differ diff --git a/assets/incremental/test1/metadata/snap-8895755745308925611-1-2e224b4d-1419-4378-b5ae-76f3ee5e039d.avro b/assets/incremental/test1/metadata/snap-8895755745308925611-1-2e224b4d-1419-4378-b5ae-76f3ee5e039d.avro new file mode 100644 index 0000000..7047144 Binary files /dev/null and b/assets/incremental/test1/metadata/snap-8895755745308925611-1-2e224b4d-1419-4378-b5ae-76f3ee5e039d.avro differ diff --git a/assets/rest/iceberg_catalog.db b/assets/rest/iceberg_catalog.db index 7e76c37..321a034 100644 Binary files a/assets/rest/iceberg_catalog.db and b/assets/rest/iceberg_catalog.db differ diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml index 67e4e55..6ebbfb0 100644 --- a/docker/docker-compose.yml +++ b/docker/docker-compose.yml @@ -65,6 +65,7 @@ services: iceberg_net: volumes: - ../assets/tpch:/input + - ../assets/incremental:/input_incremental environment: - AWS_ACCESS_KEY_ID=root - AWS_SECRET_ACCESS_KEY=password @@ -75,5 +76,6 @@ services: /usr/bin/mc rm -r --force minio/warehouse /usr/bin/mc mb minio/warehouse; /usr/bin/mc cp -r /input/tpch.sf01/ minio/warehouse/tpch.sf01/; + /usr/bin/mc cp -r /input_incremental/ minio/warehouse/incremental/; tail -f /dev/null " \ No newline at end of file diff --git a/iceberg_rust_ffi/Cargo.lock b/iceberg_rust_ffi/Cargo.lock index 2d85e47..614844e 100644 --- a/iceberg_rust_ffi/Cargo.lock +++ b/iceberg_rust_ffi/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "addr2line" -version = "0.24.2" +version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dfbe277e56a376000877090da837660b4427aad530e3028d44e0bffe4f89a1c1" +checksum = "1b5d307320b3181d6d7954e663bd7c774a838b8220fe0593c86d9fb09f498b4b" dependencies = [ "gimli", ] @@ -17,12 +17,6 @@ version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" -[[package]] -name = "adler32" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" - [[package]] name = "ahash" version = "0.7.8" @@ -42,7 +36,7 @@ checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", "const-random", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -50,9 +44,9 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] @@ -72,18 +66,6 @@ dependencies = [ "alloc-no-stdlib", ] -[[package]] -name = "allocator-api2" -version = "0.2.21" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" - -[[package]] -name = "android-tzdata" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" - [[package]] name = "android_system_properties" version = "0.1.5" @@ -95,35 +77,36 @@ dependencies = [ [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" dependencies = [ "backtrace", ] [[package]] name = "apache-avro" -version = "0.17.0" +version = "0.20.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1aef82843a0ec9f8b19567445ad2421ceeb1d711514384bdd3d49fe37102ee13" +checksum = "3a033b4ced7c585199fb78ef50fca7fe2f444369ec48080c5fd072efa1a03cc7" dependencies = [ "bigdecimal", + "bon", "digest", - "libflate", "log", + "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.8.5", + "rand 0.9.2", "regex-lite", "serde", "serde_bytes", "serde_json", - "strum 0.26.3", - "strum_macros 0.26.4", - "thiserror 1.0.69", - "typed-builder 0.19.1", + "strum", + "strum_macros", + "thiserror 2.0.17", "uuid", + "zstd", ] [[package]] @@ -140,9 +123,9 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow-arith" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30feb679425110209ae35c3fbf82404a39a4c0436bb3ec36164d8bffed2a4ce4" +checksum = "ad08897b81588f60ba983e3ca39bda2b179bdd84dced378e7df81a5313802ef8" dependencies = [ "arrow-array", "arrow-buffer", @@ -154,9 +137,9 @@ dependencies = [ [[package]] name = "arrow-array" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70732f04d285d49054a48b72c54f791bb3424abae92d27aafdf776c98af161c8" +checksum = "8548ca7c070d8db9ce7aa43f37393e4bfcf3f2d3681df278490772fd1673d08d" dependencies = [ "ahash 0.8.12", "arrow-buffer", @@ -164,15 +147,15 @@ dependencies = [ "arrow-schema", "chrono", "half", - "hashbrown 0.15.4", + "hashbrown 0.16.0", "num", ] [[package]] name = "arrow-buffer" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "169b1d5d6cb390dd92ce582b06b23815c7953e9dfaaea75556e89d890d19993d" +checksum = "e003216336f70446457e280807a73899dd822feaf02087d31febca1363e2fccc" dependencies = [ "bytes", "half", @@ -181,9 +164,9 @@ dependencies = [ [[package]] name = "arrow-cast" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e4f12eccc3e1c05a766cafb31f6a60a46c2f8efec9b74c6e0648766d30686af8" +checksum = "919418a0681298d3a77d1a315f625916cb5678ad0d74b9c60108eb15fd083023" dependencies = [ "arrow-array", "arrow-buffer", @@ -201,9 +184,9 @@ dependencies = [ [[package]] name = "arrow-data" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8de1ce212d803199684b658fc4ba55fb2d7e87b213de5af415308d2fee3619c2" +checksum = "a5c64fff1d142f833d78897a772f2e5b55b36cb3e6320376f0961ab0db7bd6d0" dependencies = [ "arrow-buffer", "arrow-schema", @@ -213,22 +196,23 @@ dependencies = [ [[package]] name = "arrow-ipc" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9ea5967e8b2af39aff5d9de2197df16e305f47f404781d3230b2dc672da5d92" +checksum = "1d3594dcddccc7f20fd069bc8e9828ce37220372680ff638c5e00dea427d88f5" dependencies = [ "arrow-array", "arrow-buffer", "arrow-data", "arrow-schema", + "arrow-select", "flatbuffers", ] [[package]] name = "arrow-ord" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6506e3a059e3be23023f587f79c82ef0bcf6d293587e3272d20f2d30b969b5a7" +checksum = "3c8f82583eb4f8d84d4ee55fd1cb306720cddead7596edce95b50ee418edf66f" dependencies = [ "arrow-array", "arrow-buffer", @@ -239,15 +223,15 @@ dependencies = [ [[package]] name = "arrow-schema" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af7686986a3bf2254c9fb130c623cdcb2f8e1f15763e7c71c310f0834da3d292" +checksum = "b3aa9e59c611ebc291c28582077ef25c97f1975383f1479b12f3b9ffee2ffabe" [[package]] name = "arrow-select" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd2b45757d6a2373faa3352d02ff5b54b098f5e21dccebc45a21806bc34501e5" +checksum = "8c41dbbd1e97bfcaee4fcb30e29105fb2c75e4d82ae4de70b792a5d3f66b2e7a" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -259,9 +243,9 @@ dependencies = [ [[package]] name = "arrow-string" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0377d532850babb4d927a06294314b316e23311503ed580ec6ce6a0158f49d40" +checksum = "53f5183c150fbc619eede22b861ea7c0eebed8eaac0333eaa7f6da5205fd504d" dependencies = [ "arrow-array", "arrow-buffer", @@ -271,7 +255,7 @@ dependencies = [ "memchr", "num", "regex", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] @@ -293,9 +277,9 @@ dependencies = [ [[package]] name = "async-compression" -version = "0.4.30" +version = "0.4.32" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "977eb15ea9efd848bb8a4a1a2500347ed7f0bf794edf0dc3ddcf439f43d36b23" +checksum = "5a89bce6054c720275ac2432fbba080a66a2106a44a1b804553930ca6909f4e0" dependencies = [ "compression-codecs", "compression-core", @@ -306,24 +290,24 @@ dependencies = [ [[package]] name = "async-lock" -version = "3.4.0" +version = "3.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ff6e472cdea888a4bd64f342f09b3f50e1886d32afe8df3d663c01140b811b18" +checksum = "5fd03604047cee9b6ce9de9f70c6cd540a0520c813cbd49bae61f33ab80ed1dc" dependencies = [ - "event-listener 5.4.0", + "event-listener 5.4.1", "event-listener-strategy", "pin-project-lite", ] [[package]] name = "async-trait" -version = "0.1.88" +version = "0.1.89" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e539d3fca749fcee5236ab05e93a52867dd549cc157c8cb7f99595f3cedffdb5" +checksum = "9035ad2d096bed7955a320ee7e2230574d28fd3c3a0f186cbea1ff3c7eed5dbb" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -360,9 +344,9 @@ dependencies = [ [[package]] name = "backon" -version = "1.5.2" +version = "1.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "592277618714fbcecda9a02ba7a8781f319d26532a88553bbacc77ba5d2b3a8d" +checksum = "cffb0e931875b666fc4fcb20fee52e9bbd1ef836fd9e9e04ec21555f9f85f7ef" dependencies = [ "fastrand", "gloo-timers", @@ -371,9 +355,9 @@ dependencies = [ [[package]] name = "backtrace" -version = "0.3.75" +version = "0.3.76" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6806a6321ec58106fea15becdad98371e28d92ccbc7c8f1b3b6dd724fe8f1002" +checksum = "bb531853791a215d7c62a30daf0dde835f381ab5de4589cfe7c649d2cbe92bd6" dependencies = [ "addr2line", "cfg-if", @@ -381,7 +365,7 @@ dependencies = [ "miniz_oxide", "object", "rustc-demangle", - "windows-targets 0.52.6", + "windows-link", ] [[package]] @@ -392,9 +376,9 @@ checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" [[package]] name = "bigdecimal" -version = "0.4.8" +version = "0.4.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a22f228ab7a1b23027ccc6c350b72868017af7ea8356fbdf19f8d991c690013" +checksum = "560f42649de9fa436b73517378a147ec21f6c997a546581df4b4b31677828934" dependencies = [ "autocfg", "libm", @@ -412,9 +396,9 @@ checksum = "230c5f1ca6a325a32553f8640d31ac9b49f2411e901e427570154868b46da4f7" [[package]] name = "bitflags" -version = "2.9.1" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b8e56985ec62d17e9c1001dc89c88ecd7dc08e47eba5ec7c29c7b5eeecde967" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "bitvec" @@ -437,6 +421,31 @@ dependencies = [ "generic-array", ] +[[package]] +name = "bon" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ebeb9aaf9329dff6ceb65c689ca3db33dbf15f324909c60e4e5eef5701ce31b1" +dependencies = [ + "bon-macros", + "rustversion", +] + +[[package]] +name = "bon-macros" +version = "3.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77e9d642a7e3a318e37c2c9427b5a6a48aa1ad55dcd986f3034ab2239045a645" +dependencies = [ + "darling 0.21.3", + "ident_case", + "prettyplease", + "proc-macro2", + "quote", + "rustversion", + "syn 2.0.108", +] + [[package]] name = "borsh" version = "1.5.7" @@ -457,14 +466,14 @@ dependencies = [ "proc-macro-crate", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "brotli" -version = "8.0.1" +version = "8.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9991eea70ea4f293524138648e41ee89b0b2b12ddef3b255effa43c8056e0e0d" +checksum = "4bd8b9603c7aa97359dbd97ecf258968c95f3adddd6db2f7e7a5bef101c84560" dependencies = [ "alloc-no-stdlib", "alloc-stdlib", @@ -511,9 +520,9 @@ dependencies = [ [[package]] name = "bytemuck" -version = "1.23.1" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c76a5792e44e4abe34d3abf15636779261d45a7450612059293d1d2cfc63422" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] name = "byteorder" @@ -529,10 +538,11 @@ checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a" [[package]] name = "cc" -version = "1.2.30" +version = "1.2.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "deec109607ca693028562ed836a5f1c4b8bd77755c4e132fc5ce11b0b6211ae7" +checksum = "37521ac7aabe3d13122dc382493e20c9416f299d2ccd5b3a5340a2570cdeb0f3" dependencies = [ + "find-msvc-tools", "jobserver", "libc", "shlex", @@ -540,9 +550,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.1" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9555578bc9e57714c812a1f84e4fc5b4d21fcb063490c624de019f7464c91268" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -552,11 +562,10 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", @@ -576,9 +585,9 @@ dependencies = [ [[package]] name = "compression-codecs" -version = "0.4.30" +version = "0.4.31" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "485abf41ac0c8047c07c87c72c8fb3eb5197f6e9d7ded615dfd1a00ae00a0f64" +checksum = "ef8a506ec4b81c460798f572caead636d57d3d7e940f998160f52bd254bf2d23" dependencies = [ "compression-core", "flate2", @@ -644,15 +653,6 @@ version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" -[[package]] -name = "core2" -version = "0.4.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b49ba7ef1ad6107f8824dbe97de947cbaac53c44e7f9756a1fba0d37c1eec505" -dependencies = [ - "memchr", -] - [[package]] name = "cpufeatures" version = "0.2.17" @@ -680,6 +680,12 @@ dependencies = [ "cfg-if", ] +[[package]] +name = "critical-section" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "790eea4361631c5e7d22598ecd5723ff611904e3344ce8720784c93e3d83d40b" + [[package]] name = "crossbeam-channel" version = "0.5.15" @@ -735,8 +741,18 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc7f46116c46ff9ab3eb1597a45688b6715c6e628b5c133e288e709a29bcb4ee" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.20.11", + "darling_macro 0.20.11", +] + +[[package]] +name = "darling" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" +dependencies = [ + "darling_core 0.21.3", + "darling_macro 0.21.3", ] [[package]] @@ -750,7 +766,21 @@ dependencies = [ "proc-macro2", "quote", "strsim", - "syn 2.0.104", + "syn 2.0.108", +] + +[[package]] +name = "darling_core" +version = "0.21.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1247195ecd7e3c85f83c8d2a366e4210d588e802133e1e355180a9870b517ea4" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim", + "syn 2.0.108", ] [[package]] @@ -759,16 +789,21 @@ version = "0.20.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc34b93ccb385b40dc71c6fceac4b2ad23662c7eeb248cf10d529b7e055b6ead" dependencies = [ - "darling_core", + "darling_core 0.20.11", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] -name = "dary_heap" -version = "0.3.7" +name = "darling_macro" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04d2cd9c18b9f454ed67da600630b021a8a80bf33f8c95896ab33aaf1c26b728" +checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" +dependencies = [ + "darling_core 0.21.3", + "quote", + "syn 2.0.108", +] [[package]] name = "data-encoding" @@ -778,12 +813,12 @@ checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" [[package]] name = "deranged" -version = "0.4.0" +version = "0.5.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c9e6a11ca8224451684bc0d7d5a7adbf8f2fd6887261a1cfc3c0432f9d4068e" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" dependencies = [ "powerfmt", - "serde", + "serde_core", ] [[package]] @@ -801,10 +836,10 @@ version = "0.20.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2d5bcf7b024d6835cfb3d473887cd966994907effbe9227e8c8219824d06c4e8" dependencies = [ - "darling", + "darling 0.20.11", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -814,7 +849,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ab63b0e2bf4d5928aff72e83a7dace85d7bba5fe12dcc3c5a572d78caffd3f3c" dependencies = [ "derive_builder_core", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -837,7 +872,7 @@ checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -882,7 +917,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -893,12 +928,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "778e2ac28f6c47af28e4907f13ffd1e1ddbd400980a9abd7c8df189bf578a5ad" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] @@ -909,9 +944,9 @@ checksum = "0206175f82b8d6bf6652ff7d71a1e27fd2e4efde587fd368662814d6ec1d9ce0" [[package]] name = "event-listener" -version = "5.4.0" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3492acde4c3fc54c845eaab3eed8bd00c7a7d881f78bfc801e43a93dec1331ae" +checksum = "e13b66accf52311f30a0db42147dadea9850cb48cd070028831ae5f5d4b856ab" dependencies = [ "concurrent-queue", "parking", @@ -924,7 +959,7 @@ version = "0.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8be9f3dfaaffdae2972880079a491a1a8bb7cbed0b8dd7a347f668b4150a3b93" dependencies = [ - "event-listener 5.4.0", + "event-listener 5.4.1", "pin-project-lite", ] @@ -944,11 +979,17 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" +[[package]] +name = "find-msvc-tools" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "52051878f80a721bb68ebfbc930e07b65ba72f2da88968ea5c06fd6ca3d3a127" + [[package]] name = "flatbuffers" -version = "25.2.10" +version = "25.9.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1045398c1bfd89168b5fd3f1fc11f6e70b34f6f66300c87d44d3de849463abf1" +checksum = "09b6620799e7340ebd9968d2e0708eb82cf1971e9a16821e2091b6d6e475eed5" dependencies = [ "bitflags", "rustc_version", @@ -956,9 +997,9 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.2" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a3d7db9596fecd151c5f638c0ee5d5bd487b6e0ea232e5dc96d5250f6f94b1d" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "libz-ng-sys", @@ -1001,9 +1042,9 @@ checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" [[package]] name = "form_urlencoded" -version = "1.2.1" +version = "1.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e13624c2627564efccf4934284bdd98cbaa14e79b0b5a141218e507b3a823456" +checksum = "cb4cb245038516f5f85277875cdaa4f7d2c9a0fa0468de06ed190163b1581fcf" dependencies = [ "percent-encoding", ] @@ -1070,7 +1111,7 @@ checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -1103,25 +1144,11 @@ dependencies = [ "slab", ] -[[package]] -name = "generator" -version = "0.8.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d18470a76cb7f8ff746cf1f7470914f900252ec36bbc40b569d74b1258446827" -dependencies = [ - "cc", - "cfg-if", - "libc", - "log", - "rustversion", - "windows", -] - [[package]] name = "generic-array" -version = "0.14.7" +version = "0.14.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85649ca51fd72272d7821adaf274ad91c288277713d9c18820d8499a7ff69e9a" +checksum = "4bb6743198531e02858aeaea5398fcc883e71851fcbcb5a2f773e2fb6cb1edf2" dependencies = [ "typenum", "version_check", @@ -1136,29 +1163,29 @@ dependencies = [ "cfg-if", "js-sys", "libc", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "wasm-bindgen", ] [[package]] name = "getrandom" -version = "0.3.3" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.31.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" [[package]] name = "gloo-timers" @@ -1184,7 +1211,7 @@ dependencies = [ "futures-core", "futures-sink", "http", - "indexmap 2.10.0", + "indexmap 2.12.0", "slab", "tokio", "tokio-util", @@ -1193,13 +1220,14 @@ dependencies = [ [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", "num-traits", + "zerocopy", ] [[package]] @@ -1218,14 +1246,13 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e5274423e17b7c9fc20b6e7e208532f9b19825d82dfd615708b70edd83df41f1" dependencies = [ "ahash 0.8.12", - "allocator-api2", ] [[package]] name = "hashbrown" -version = "0.15.4" +version = "0.16.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5971ac85611da7067dbfcabef3c70ebb5606018acd9e2a3903a0da507521e0d5" +checksum = "5419bdc4f6a9207fbeba6d11b604d481addf78ecd10c11ad51e76c2f6482748d" [[package]] name = "heck" @@ -1269,6 +1296,31 @@ dependencies = [ "url", ] +[[package]] +name = "hickory-proto" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8a6fe56c0038198998a6f217ca4e7ef3a5e51f46163bd6dd60b5c71ca6c6502" +dependencies = [ + "async-trait", + "cfg-if", + "data-encoding", + "enum-as-inner", + "futures-channel", + "futures-io", + "futures-util", + "idna", + "ipnet", + "once_cell", + "rand 0.9.2", + "ring", + "thiserror 2.0.17", + "tinyvec", + "tokio", + "tracing", + "url", +] + [[package]] name = "hickory-resolver" version = "0.24.4" @@ -1277,7 +1329,7 @@ checksum = "cbb117a1ca520e111743ab2f6688eddee69db4e0ea242545a604dce8a66fd22e" dependencies = [ "cfg-if", "futures-util", - "hickory-proto", + "hickory-proto 0.24.4", "ipconfig", "lru-cache", "once_cell", @@ -1290,6 +1342,27 @@ dependencies = [ "tracing", ] +[[package]] +name = "hickory-resolver" +version = "0.25.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dc62a9a99b0bfb44d2ab95a7208ac952d31060efc16241c87eaf36406fecf87a" +dependencies = [ + "cfg-if", + "futures-util", + "hickory-proto 0.25.2", + "ipconfig", + "moka", + "once_cell", + "parking_lot", + "rand 0.9.2", + "resolv-conf", + "smallvec", + "thiserror 2.0.17", + "tokio", + "tracing", +] + [[package]] name = "hmac" version = "0.12.1" @@ -1301,11 +1374,11 @@ dependencies = [ [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -1350,25 +1423,27 @@ checksum = "6dbf3de79e51f3d586ab4cb9d5c3e2c14aa28ed23d180cf89b4df0454a69cc87" [[package]] name = "humantime" -version = "2.2.0" +version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9b112acc8b3adf4b107a8ec20977da0273a8c386765a3ec0229bd500a1443f9f" +checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" [[package]] name = "hyper" -version = "1.6.0" +version = "1.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cc2b571658e38e0c01b1fdca3bbbe93c00d3d71693ff2770043f8c29bc7d6f80" +checksum = "eb3aa54a13a0dfe7fbe3a59e0c76093041720fdc77b110cc0fc260fafb4dc51e" dependencies = [ + "atomic-waker", "bytes", "futures-channel", - "futures-util", + "futures-core", "h2", "http", "http-body", "httparse", "itoa", "pin-project-lite", + "pin-utils", "smallvec", "tokio", "want", @@ -1394,9 +1469,9 @@ dependencies = [ [[package]] name = "hyper-util" -version = "0.1.16" +version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d9b05277c7e8da2c93a568989bb6207bef0112e8d17df7a6eda4a3cf143bc5e" +checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ "base64", "bytes", @@ -1410,7 +1485,7 @@ dependencies = [ "libc", "percent-encoding", "pin-project-lite", - "socket2 0.6.0", + "socket2 0.6.1", "tokio", "tower-service", "tracing", @@ -1418,9 +1493,9 @@ dependencies = [ [[package]] name = "iana-time-zone" -version = "0.1.63" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1442,9 +1517,8 @@ dependencies = [ [[package]] name = "iceberg" -version = "0.6.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "306fd4bf70d30687dc765110ecd19fc2bb21f16c3d5c188bc53a0d573bb6e675" +version = "0.7.0" +source = "git+https://github.com/RelationalAI/iceberg-rust.git?rev=ccb1e0c9983a1bdcb5b70fa637759df526a9a75e#ccb1e0c9983a1bdcb5b70fa637759df526a9a75e" dependencies = [ "anyhow", "apache-avro", @@ -1477,6 +1551,7 @@ dependencies = [ "ordered-float 4.6.0", "parquet", "rand 0.8.5", + "reqsign", "reqwest", "roaring", "rust_decimal", @@ -1486,10 +1561,10 @@ dependencies = [ "serde_json", "serde_repr", "serde_with", - "strum 0.27.2", + "strum", "thrift", "tokio", - "typed-builder 0.20.1", + "typed-builder", "url", "uuid", "zstd", @@ -1515,9 +1590,9 @@ dependencies = [ [[package]] name = "icu_collections" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "200072f5d0e3614556f94a9930d5dc3e0662a652823904c3a75dc3b0af7fee47" +checksum = "4c6b649701667bbe825c3b7e6388cb521c23d88644678e83c0c4d0a621a34b43" dependencies = [ "displaydoc", "potential_utf", @@ -1528,9 +1603,9 @@ dependencies = [ [[package]] name = "icu_locale_core" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0cde2700ccaed3872079a65fb1a78f6c0a36c91570f28755dda67bc8f7d9f00a" +checksum = "edba7861004dd3714265b4db54a3c390e880ab658fec5f7db895fae2046b5bb6" dependencies = [ "displaydoc", "litemap", @@ -1541,11 +1616,10 @@ dependencies = [ [[package]] name = "icu_normalizer" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "436880e8e18df4d7bbc06d58432329d6458cc84531f7ac5f024e93deadb37979" +checksum = "5f6c8828b67bf8908d82127b2054ea1b4427ff0230ee9141c54251934ab1b599" dependencies = [ - "displaydoc", "icu_collections", "icu_normalizer_data", "icu_properties", @@ -1556,42 +1630,38 @@ dependencies = [ [[package]] name = "icu_normalizer_data" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "00210d6893afc98edb752b664b8890f0ef174c8adbb8d0be9710fa66fbbf72d3" +checksum = "7aedcccd01fc5fe81e6b489c15b247b8b0690feb23304303a9e560f37efc560a" [[package]] name = "icu_properties" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "016c619c1eeb94efb86809b015c58f479963de65bdb6253345c1a1276f22e32b" +checksum = "e93fcd3157766c0c8da2f8cff6ce651a31f0810eaa1c51ec363ef790bbb5fb99" dependencies = [ - "displaydoc", "icu_collections", "icu_locale_core", "icu_properties_data", "icu_provider", - "potential_utf", "zerotrie", "zerovec", ] [[package]] name = "icu_properties_data" -version = "2.0.1" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "298459143998310acd25ffe6810ed544932242d3f07083eee1084d83a71bd632" +checksum = "02845b3647bb045f1100ecd6480ff52f34c35f82d9880e029d329c21d1054899" [[package]] name = "icu_provider" -version = "2.0.0" +version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03c80da27b5f4187909049ee2d72f276f0d9f99a42c306bd0131ecfe04d8e5af" +checksum = "85962cf0ce02e1e0a629cc34e7ca3e373ce20dda4c4d7294bbd0bf1fdb59e614" dependencies = [ "displaydoc", "icu_locale_core", - "stable_deref_trait", - "tinystr", "writeable", "yoke", "zerofrom", @@ -1607,9 +1677,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "idna" -version = "1.0.3" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "686f825264d630750a544639377bae737628043f20d38bbc029e8f29ea968a7e" +checksum = "3b0875f23caa03898994f6ddc501886a45c7d3d62d04d2d90788d47be1b1e4de" dependencies = [ "idna_adapter", "smallvec", @@ -1639,13 +1709,14 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.10.0" +version = "2.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fe4cd85333e22411419a0bcae1297d25e58c9443848b11dc6a86fefe8c78a661" +checksum = "6717a8d2a5a929a1a2eb43a12812498ed141a0bcfb7e8f7844fbdbe4303bba9f" dependencies = [ "equivalent", - "hashbrown 0.15.4", + "hashbrown 0.16.0", "serde", + "serde_core", ] [[package]] @@ -1663,17 +1734,6 @@ version = "3.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8bb03732005da905c88227371639bf1ad885cc712789c011c31c5fb3ab3ccf02" -[[package]] -name = "io-uring" -version = "0.7.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d93587f37623a1a17d94ef2bc9ada592f5465fe7732084ab7beefabe5c77c0c4" -dependencies = [ - "bitflags", - "cfg-if", - "libc", -] - [[package]] name = "ipconfig" version = "0.3.2" @@ -1719,19 +1779,19 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jobserver" -version = "0.1.33" +version = "0.1.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38f262f097c174adebe41eb73d66ae9c06b2844fb0da69969647bbddd9b0538a" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "libc", ] [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "b011eec8cc36da2aab2d5cff675ec18454fad408585853910a202391cf9f8e65" dependencies = [ "once_cell", "wasm-bindgen", @@ -1745,9 +1805,9 @@ checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" [[package]] name = "lexical-core" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b765c31809609075565a70b4b71402281283aeda7ecaf4818ac14a7b2ade8958" +checksum = "7d8d125a277f807e55a77304455eb7b1cb52f2b18c143b60e766c120bd64a594" dependencies = [ "lexical-parse-float", "lexical-parse-integer", @@ -1758,84 +1818,53 @@ dependencies = [ [[package]] name = "lexical-parse-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexical-write-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5afc668a27f460fb45a81a757b6bf2f43c2d7e30cb5a2dcd3abf294c78d62bd" +checksum = "50c438c87c013188d415fbabbb1dceb44249ab81664efbd31b14ae55dabb6361" dependencies = [ "lexical-util", "lexical-write-integer", - "static_assertions", ] [[package]] name = "lexical-write-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "629ddff1a914a836fb245616a7888b62903aae58fa771e1d83943035efa0f978" +checksum = "409851a618475d2d5796377cad353802345cba92c867d9fbcde9cf4eac4e14df" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "libc" -version = "0.2.174" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1171693293099992e19cddea4e8b849964e9846f4acee11b3948bcc337be8776" - -[[package]] -name = "libflate" -version = "2.1.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45d9dfdc14ea4ef0900c1cddbc8dcd553fbaacd8a4a282cf4018ae9dd04fb21e" -dependencies = [ - "adler32", - "core2", - "crc32fast", - "dary_heap", - "libflate_lz77", -] - -[[package]] -name = "libflate_lz77" -version = "2.1.0" +version = "0.2.177" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e0d73b369f386f1c44abd9c570d5318f55ccde816ff4b562fa452e5182863d" -dependencies = [ - "core2", - "hashbrown 0.14.5", - "rle-decode-fast", -] +checksum = "2874a2af47a2325c2001a6e6fad9b16a53b802102b528163885171cf92b15976" [[package]] name = "libm" @@ -1855,9 +1884,9 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "172a788537a2221661b480fee8dc5f96c580eb34fa88764d3205dc356c7e4221" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" dependencies = [ "zlib-rs", ] @@ -1870,44 +1899,30 @@ checksum = "0717cef1bc8b636c6e1c1bbdefc09e6322da8a9321966e8928ef80d20f7f770f" [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "litemap" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +checksum = "6373607a59f0be73a39b6fe456b8192fcc3585f602af20751600e974dd455e77" [[package]] name = "lock_api" -version = "0.4.13" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "96936507f153605bddfcda068dd804796c84324ed2510809e5b2a624c81da765" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.27" +version = "0.4.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" - -[[package]] -name = "loom" -version = "0.7.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" -dependencies = [ - "cfg-if", - "generator", - "scoped-tls", - "tracing", - "tracing-subscriber", -] +checksum = "34080505efa8e45a4b816c349525ebe327ceaa8559756f0356cba97ef3bf7432" [[package]] name = "lru-cache" @@ -1935,11 +1950,11 @@ dependencies = [ [[package]] name = "matchers" -version = "0.1.0" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8263075bb86c5a1b1427b5ae862e8889656f126e9f77c484496e8b47cf5c5558" +checksum = "d1525a2a28c7f4fa0fc98bb91ae755d1e2d1505079e05539e35bc876b5d65ae9" dependencies = [ - "regex-automata 0.1.10", + "regex-automata", ] [[package]] @@ -1954,9 +1969,9 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.5" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "metrics" @@ -1978,7 +1993,7 @@ dependencies = [ "crossbeam-epoch", "crossbeam-utils", "hashbrown 0.14.5", - "indexmap 2.10.0", + "indexmap 2.12.0", "metrics", "num_cpus", "ordered-float 4.6.0", @@ -1994,38 +2009,38 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] name = "mio" -version = "1.0.4" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78bed444cc8a2160f01cbcf811ef18cac863ad68ae8ca62092e8db51d51c761c" +checksum = "69d83b0086dc8ecf3ce9ae2874b2d1290252e2a30720bea58a5c6639b0092873" dependencies = [ "libc", - "wasi 0.11.1+wasi-snapshot-preview1", - "windows-sys 0.59.0", + "wasi", + "windows-sys 0.61.2", ] [[package]] name = "moka" -version = "0.12.10" +version = "0.12.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9321642ca94a4282428e6ea4af8cc2ca4eac48ac7a6a4ea8f33f76d0ce70926" +checksum = "8261cd88c312e0004c1d51baad2980c66528dfdb2bee62003e643a4d8f86b077" dependencies = [ "async-lock", "crossbeam-channel", "crossbeam-epoch", "crossbeam-utils", - "event-listener 5.4.0", + "equivalent", + "event-listener 5.4.1", "futures-util", - "loom", "parking_lot", "portable-atomic", "rustc_version", "smallvec", "tagptr", - "thiserror 1.0.69", "uuid", ] @@ -2055,12 +2070,11 @@ dependencies = [ [[package]] name = "nu-ansi-term" -version = "0.46.0" +version = "0.50.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77a8165726e8236064dbb45459242600304b42a5ea24ee2948e18e023bf7ba84" +checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5" dependencies = [ - "overload", - "winapi", + "windows-sys 0.61.2", ] [[package]] @@ -2156,9 +2170,9 @@ dependencies = [ [[package]] name = "object" -version = "0.36.7" +version = "0.37.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "62948e14d923ea95ea2c7c86c71013138b66525b86bdc08d2dcc262bdb497b87" +checksum = "ff76201f031d8863c38aa7f905eca4f53abbfa15f609db4277d44cd8938f33fe" dependencies = [ "memchr", ] @@ -2180,7 +2194,7 @@ dependencies = [ "md-5", "parking_lot", "percent-encoding", - "quick-xml", + "quick-xml 0.37.5", "rand 0.8.5", "reqwest", "ring", @@ -2210,7 +2224,7 @@ dependencies = [ "flate2", "flume", "futures-util", - "hickory-resolver", + "hickory-resolver 0.24.4", "hyper", "metrics", "metrics-util", @@ -2242,12 +2256,16 @@ name = "once_cell" version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +dependencies = [ + "critical-section", + "portable-atomic", +] [[package]] name = "opendal" -version = "0.54.0" +version = "0.54.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ffb9838d0575c6dbaf3fcec7255af8d5771996d4af900bbb6fa9a314dec00a1a" +checksum = "42afda58fa2cf50914402d132cc1caacff116a85d10c72ab2082bb7c50021754" dependencies = [ "anyhow", "backon", @@ -2262,7 +2280,7 @@ dependencies = [ "log", "md-5", "percent-encoding", - "quick-xml", + "quick-xml 0.38.3", "reqsign", "reqwest", "serde", @@ -2273,9 +2291,9 @@ dependencies = [ [[package]] name = "openssl" -version = "0.10.73" +version = "0.10.74" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8505734d46c8ab1e19a1dce3aef597ad87dcb4c37e7188231769bd6bd51cebf8" +checksum = "24ad14dd45412269e1a30f52ad8f0664f0f4f4a89ee8fe28c3b3527021ebb654" dependencies = [ "bitflags", "cfg-if", @@ -2294,7 +2312,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -2305,18 +2323,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.5.2+3.5.2" +version = "300.5.4+3.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d270b79e2926f5150189d475bc7e9d2c69f9c4697b185fa917d5a32b792d21b4" +checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.109" +version = "0.9.110" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "90096e2e47630d78b7d1c20952dc621f957103f8bc2c8359ec81290d75238571" +checksum = "0a9f0075ba3c21b09f8e8b2026584b1d18d49388648f2fbbf3c97ea8deced8e2" dependencies = [ "cc", "libc", @@ -2353,12 +2371,6 @@ dependencies = [ "hashbrown 0.14.5", ] -[[package]] -name = "overload" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b15813163c1d831bf4a13c3610c05c0d03b39feb07f7e09fa234dac9b15aaf39" - [[package]] name = "parking" version = "2.2.1" @@ -2367,9 +2379,9 @@ checksum = "f38d5652c16fde515bb1ecef450ab0f6a219d619a7274976324d5e377f7dceba" [[package]] name = "parking_lot" -version = "0.12.4" +version = "0.12.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "70d58bf43669b5795d1576d0641cfb6fbb2057bf629506267a92807158584a13" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" dependencies = [ "lock_api", "parking_lot_core", @@ -2377,22 +2389,22 @@ dependencies = [ [[package]] name = "parking_lot_core" -version = "0.9.11" +version = "0.9.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" dependencies = [ "cfg-if", "libc", "redox_syscall", "smallvec", - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "parquet" -version = "55.2.0" +version = "56.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b17da4150748086bd43352bc77372efa9b6e3dbd06a04831d2a98c041c225cfa" +checksum = "f0dbd48ad52d7dccf8ea1b90a3ddbfaea4f69878dd7683e51c507d4bc52b5b27" dependencies = [ "ahash 0.8.12", "arrow-array", @@ -2409,7 +2421,7 @@ dependencies = [ "flate2", "futures", "half", - "hashbrown 0.15.4", + "hashbrown 0.16.0", "lz4_flex", "num", "num-bigint", @@ -2431,9 +2443,9 @@ checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" [[package]] name = "percent-encoding" -version = "2.3.1" +version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3148f5046208a5d56bcfc03053e3ca6334e51da8dfb19b6cdc8b306fae3283e" +checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" [[package]] name = "pin-project" @@ -2452,7 +2464,7 @@ checksum = "6e918e4ff8c4549eb882f14b3a4bc8c8bc93de829416eacf579f1207a8fbf861" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -2481,9 +2493,9 @@ checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "potential_utf" -version = "0.1.2" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5a7c30837279ca13e7c867e9e40053bc68740f988cb07f7ca6df43cc734b585" +checksum = "b73949432f5e2a09657003c25bca5e19a0e9c84f8058ca374f49e0ebe605af77" dependencies = [ "zerovec", ] @@ -2503,20 +2515,30 @@ dependencies = [ "zerocopy", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn 2.0.108", +] + [[package]] name = "proc-macro-crate" -version = "3.3.0" +version = "3.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "edce586971a4dfaa28950c6f18ed55e0406c1ab88bbce2c6f6293a7aaba73d35" +checksum = "219cb19e96be00ab2e37d6e299658a0cfa83e52429179969b0f0121b4ac46983" dependencies = [ "toml_edit", ] [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -2557,7 +2579,7 @@ dependencies = [ "libc", "once_cell", "raw-cpuid", - "wasi 0.11.1+wasi-snapshot-preview1", + "wasi", "web-sys", "winapi", ] @@ -2572,11 +2594,21 @@ dependencies = [ "serde", ] +[[package]] +name = "quick-xml" +version = "0.38.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42a232e7487fc2ef313d96dde7948e7a3c05101870d8985e4fd8d26aedd27b89" +dependencies = [ + "memchr", + "serde", +] + [[package]] name = "quinn" -version = "0.11.8" +version = "0.11.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626214629cda6781b6dc1d316ba307189c85ba657213ce642d9c77670f8202c8" +checksum = "b9e20a958963c291dc322d98411f541009df2ced7b5a4f2bd52337638cfccf20" dependencies = [ "bytes", "cfg_aliases", @@ -2585,8 +2617,8 @@ dependencies = [ "quinn-udp", "rustc-hash", "rustls", - "socket2 0.5.10", - "thiserror 2.0.12", + "socket2 0.6.1", + "thiserror 2.0.17", "tokio", "tracing", "web-time", @@ -2594,12 +2626,12 @@ dependencies = [ [[package]] name = "quinn-proto" -version = "0.11.12" +version = "0.11.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e" +checksum = "f1906b49b0c3bc04b5fe5d86a77925ae6524a19b816ae38ce1e426255f1d8a31" dependencies = [ "bytes", - "getrandom 0.3.3", + "getrandom 0.3.4", "lru-slab", "rand 0.9.2", "ring", @@ -2607,7 +2639,7 @@ dependencies = [ "rustls", "rustls-pki-types", "slab", - "thiserror 2.0.12", + "thiserror 2.0.17", "tinyvec", "tracing", "web-time", @@ -2615,23 +2647,23 @@ dependencies = [ [[package]] name = "quinn-udp" -version = "0.5.13" +version = "0.5.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fcebb1209ee276352ef14ff8732e24cc2b02bbac986cd74a4c81bcb2f9881970" +checksum = "addec6a0dcad8a8d96a771f815f0eaf55f9d1805756410b39f5fa81332574cbd" dependencies = [ "cfg_aliases", "libc", "once_cell", - "socket2 0.5.10", + "socket2 0.6.1", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.41" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "ce25767e7b499d1b604768e7cde645d14cc8584231ea6b295e9c9eb22c02e1d1" dependencies = [ "proc-macro2", ] @@ -2714,7 +2746,7 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", ] [[package]] @@ -2728,82 +2760,67 @@ dependencies = [ [[package]] name = "redox_syscall" -version = "0.5.17" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5407465600fb0548f1442edf71dd20683c6ed326200ace4b1ef0763521bb3b77" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ "bitflags", ] [[package]] name = "ref-cast" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a0ae411dbe946a674d89546582cea4ba2bb8defac896622d6496f14c23ba5cf" +checksum = "f354300ae66f76f1c85c5f84693f0ce81d747e2c3f21a45fef496d89c960bf7d" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.24" +version = "1.0.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1165225c21bff1f3bbce98f5a1f889949bc902d3575308cc7b0de30b4f6d27c7" +checksum = "b7186006dcb21920990093f30e3dea63b7d6e977bf1256be20c3563a5db070da" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", - "regex-automata 0.4.9", - "regex-syntax 0.8.5", + "regex-automata", + "regex-syntax", ] [[package]] name = "regex-automata" -version = "0.1.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6c230d73fb8d8c1b9c0b3135c5142a8acee3a0558fb8db5cf1cb65f8d7862132" -dependencies = [ - "regex-syntax 0.6.29", -] - -[[package]] -name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", - "regex-syntax 0.8.5", + "regex-syntax", ] [[package]] name = "regex-lite" -version = "0.1.6" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53a49587ad06b26609c52e423de037e7f57f20d53535d66e08c695f347df952a" +checksum = "8d942b98df5e658f56f20d592c7f868833fe38115e65c33003d8cd224b0155da" [[package]] name = "regex-syntax" -version = "0.6.29" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f162c6dd7b008981e4d40210aca20b4bd0f9b60ca9271061b07f78537722f2e1" - -[[package]] -name = "regex-syntax" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "rend" @@ -2832,7 +2849,7 @@ dependencies = [ "http", "log", "percent-encoding", - "quick-xml", + "quick-xml 0.37.5", "rand 0.8.5", "reqwest", "rust-ini", @@ -2845,16 +2862,16 @@ dependencies = [ [[package]] name = "reqwest" -version = "0.12.22" +version = "0.12.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cbc931937e6ca3a06e3b6c0aa7841849b160a90351d6ab467a8b9b9959767531" +checksum = "9d0946410b9f7b082a427e4ef5c8ff541a88b357bc6c637c40db3a68ac70a36f" dependencies = [ "base64", "bytes", "futures-core", "futures-util", "h2", - "hickory-resolver", + "hickory-resolver 0.25.2", "http", "http-body", "http-body-util", @@ -2890,9 +2907,9 @@ dependencies = [ [[package]] name = "resolv-conf" -version = "0.7.4" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95325155c684b1c89f7765e30bc1c42e4a6da51ca513615660cb8a62ef9a88e3" +checksum = "6b3789b30bd25ba102de4beabd95d21ac45b69b1be7d14522bab988c526d6799" [[package]] name = "ring" @@ -2937,17 +2954,11 @@ dependencies = [ "syn 1.0.109", ] -[[package]] -name = "rle-decode-fast" -version = "1.0.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3582f63211428f83597b51b2ddb88e2a91a9d52d12831f9d08f5e624e8977422" - [[package]] name = "roaring" -version = "0.10.12" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19e8d2cfa184d94d0726d650a9f4a1be7f9b76ac9fdb954219878dc00c1c1e7b" +checksum = "f08d6a905edb32d74a5d5737a0c9d7e950c312f3c46cb0ca0a2ca09ea11878a0" dependencies = [ "bytemuck", "byteorder", @@ -2955,9 +2966,9 @@ dependencies = [ [[package]] name = "rust-ini" -version = "0.21.2" +version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7295b7ce3bf4806b419dc3420745998b447178b7005e2011947b38fc5aa6791" +checksum = "796e8d2b6696392a43bea58116b667fb4c29727dc5abd27d6acf338bb4f688c7" dependencies = [ "cfg-if", "ordered-multimap", @@ -2965,9 +2976,9 @@ dependencies = [ [[package]] name = "rust_decimal" -version = "1.37.2" +version = "1.39.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b203a6425500a03e0919c42d3c47caca51e79f1132046626d2c8871c5092035d" +checksum = "35affe401787a9bd846712274d97654355d21b2a2c092a3139aabe31e9022282" dependencies = [ "arrayvec", "borsh", @@ -3002,22 +3013,22 @@ dependencies = [ [[package]] name = "rustix" -version = "1.0.8" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "11181fbabf243db407ef8df94a6ce0b2f9a733bd8be4ad02b4eda9602296cac8" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ "bitflags", "errno", "libc", "linux-raw-sys", - "windows-sys 0.60.2", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.31" +version = "0.23.34" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ebcbd2f03de0fc1122ad9bb24b127a5a6cd51d72604a3f3c50ac459762b6cc" +checksum = "6a9586e9ee2b4f8fab52a0048ca7334d7024eef48e2cb9407e3497bb7cab7fa7" dependencies = [ "once_cell", "ring", @@ -3029,9 +3040,9 @@ dependencies = [ [[package]] name = "rustls-native-certs" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fcff2dd52b58a8d98a70243663a0d234c4e2b79235637849d15913394a247d3" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" dependencies = [ "openssl-probe", "rustls-pki-types", @@ -3041,9 +3052,9 @@ dependencies = [ [[package]] name = "rustls-pki-types" -version = "1.12.0" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "229a4a4c221013e7e1f1a043678c5cc39fe5171437c88fb47151a21e6f5b5c79" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" dependencies = [ "web-time", "zeroize", @@ -3051,9 +3062,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.4" +version = "0.103.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a17884ae0c1b773f1ccd2bd4a8c72f16da897310a98b0e84bf349ad5ead92fc" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" dependencies = [ "ring", "rustls-pki-types", @@ -3062,9 +3073,9 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.21" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a0d197bd2c9dc6e53b84da9556a69ba4cdfab8619eb41a8bd1cc2027a0f6b1d" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "ryu" @@ -3083,11 +3094,11 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3114,12 +3125,6 @@ dependencies = [ "serde_json", ] -[[package]] -name = "scoped-tls" -version = "1.0.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" - [[package]] name = "scopeguard" version = "1.2.0" @@ -3134,9 +3139,9 @@ checksum = "1c107b6f4780854c8b126e228ea8869f4d7b71260f962fefb57b996b8959ba6b" [[package]] name = "security-framework" -version = "3.3.0" +version = "3.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "80fb1d92c5028aa318b4b8bd7302a5bfcf48be96a37fc6fc790f806b0004ee0c" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" dependencies = [ "bitflags", "core-foundation", @@ -3147,9 +3152,9 @@ dependencies = [ [[package]] name = "security-framework-sys" -version = "2.14.0" +version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49db231d56a190491cb4aeda9527f1ad45345af50b0851622a7adb8c03b01c32" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ "core-foundation-sys", "libc", @@ -3157,9 +3162,9 @@ dependencies = [ [[package]] name = "semver" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6fa9c48d24d85fb3de5ad847117517440f6beceb7798af16b4a87d616b8d0" +checksum = "d767eb0aabc880b29956c35734170f26ed551a859dbd361d140cdbeca61ab1e2" [[package]] name = "seq-macro" @@ -3169,53 +3174,66 @@ checksum = "1bc711410fbe7399f390ca1c3b60ad0f53f80e95c5eb935e52268a0e2cd49acc" [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde_bytes" -version = "0.11.17" +version = "0.11.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8437fd221bde2d4ca316d61b90e337e9e702b3820b87d63caa9ba6c02bd06d96" +checksum = "a5d440709e79d88e51ac01c4b72fc6cb7314017bb7da9eeff678aa94c10e3ea8" dependencies = [ "serde", + "serde_core", +] + +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", ] [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "serde_json" -version = "1.0.141" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30b9eff21ebe718216c6ec64e1d9ac57087aad11efc64e32002bce4a0d4c03d3" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] name = "serde_path_to_error" -version = "0.1.17" +version = "0.1.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "59fab13f937fa393d08645bf3a84bdfe86e296747b506ada67bb15f10f218b2a" +checksum = "10a9ff822e371bb5403e391ecd83e182e0e77ba7f6fe0160b795797109d1b457" dependencies = [ "itoa", "serde", + "serde_core", ] [[package]] @@ -3226,7 +3244,7 @@ checksum = "175ee3e80ae9982737ca543e96133087cbd9a485eecc3bc4de9c1a37b47ea59c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -3243,19 +3261,18 @@ dependencies = [ [[package]] name = "serde_with" -version = "3.14.0" +version = "3.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2c45cd61fefa9db6f254525d46e392b852e0e61d9a1fd36e5bd183450a556d5" +checksum = "aa66c845eee442168b2c8134fec70ac50dc20e760769c8ba0ad1319ca1959b04" dependencies = [ "base64", "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.10.0", + "indexmap 2.12.0", "schemars 0.9.0", "schemars 1.0.4", - "serde", - "serde_derive", + "serde_core", "serde_json", "serde_with_macros", "time", @@ -3263,14 +3280,14 @@ dependencies = [ [[package]] name = "serde_with_macros" -version = "3.14.0" +version = "3.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de90945e6565ce0d9a25098082ed4ee4002e047cb59892c318d66821e14bb30f" +checksum = "b91a903660542fced4e99881aa481bdbaec1634568ee02e0b8bd57c64cb38955" dependencies = [ - "darling", + "darling 0.21.3", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -3312,13 +3329,19 @@ checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" [[package]] name = "signal-hook-registry" -version = "1.4.5" +version = "1.4.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9203b8055f63a2a00e2f593bb0510367fe707d7ff1e5c872de2f537b339e5410" +checksum = "b2a4719bff48cee6b39d12c020eeb490953ad2443b7055bd0b21fca26bd8c28b" dependencies = [ "libc", ] +[[package]] +name = "simd-adler32" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" + [[package]] name = "simdutf8" version = "0.1.5" @@ -3333,9 +3356,9 @@ checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" [[package]] name = "slab" -version = "0.4.10" +version = "0.4.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04dc19736151f35336d325007ac991178d504a119863a2fcb3758cdb5e52c50d" +checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589" [[package]] name = "smallvec" @@ -3361,7 +3384,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -3382,12 +3405,12 @@ dependencies = [ [[package]] name = "socket2" -version = "0.6.0" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "233504af464074f9d066d7b5416c5f9b894a5862a6506e306f7b816cdd6f1807" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -3401,15 +3424,9 @@ dependencies = [ [[package]] name = "stable_deref_trait" -version = "1.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" - -[[package]] -name = "static_assertions" -version = "1.1.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "strsim" @@ -3417,32 +3434,13 @@ version = "0.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7da8b5736845d9f2fcb837ea5d9e2628564b3b043a70948a3f0b778838c5fb4f" -[[package]] -name = "strum" -version = "0.26.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8fec0f0aef304996cf250b31b5a10dee7980c85da9d759361292b8bca5a18f06" - [[package]] name = "strum" version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" dependencies = [ - "strum_macros 0.27.2", -] - -[[package]] -name = "strum_macros" -version = "0.26.4" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4c6bee85a5a24955dc440386795aa378cd9cf82acd5f764469152d2270e581be" -dependencies = [ - "heck", - "proc-macro2", - "quote", - "rustversion", - "syn 2.0.104", + "strum_macros", ] [[package]] @@ -3454,7 +3452,7 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -3476,9 +3474,9 @@ dependencies = [ [[package]] name = "syn" -version = "2.0.104" +version = "2.0.108" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17b6f705963418cdb9927482fa304bc562ece2fdd4f616084c50b7023b435a40" +checksum = "da58917d35242480a05c2897064da0a80589a2a0476c9a3f2fdc83b53502e917" dependencies = [ "proc-macro2", "quote", @@ -3502,7 +3500,7 @@ checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -3519,15 +3517,15 @@ checksum = "55937e1799185b12863d447f42597ed69d9928686b8d88a1df17376a097d8369" [[package]] name = "tempfile" -version = "3.20.0" +version = "3.23.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8a64e3985349f2441a1a9ef0b853f869006c3855f2cda6862a94d26ebb9d6a1" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" dependencies = [ "fastrand", - "getrandom 0.3.3", + "getrandom 0.3.4", "once_cell", "rustix", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3541,11 +3539,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.17", ] [[package]] @@ -3556,18 +3554,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -3603,9 +3601,9 @@ dependencies = [ [[package]] name = "time" -version = "0.3.41" +version = "0.3.44" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8a7619e19bc266e0f9c5e6686659d394bc57973859340060a69221e57dbc0c40" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" dependencies = [ "deranged", "itoa", @@ -3618,15 +3616,15 @@ dependencies = [ [[package]] name = "time-core" -version = "0.1.4" +version = "0.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9e9a38711f559d9e3ce1cdb06dd7c5b8ea546bc90052da6d06bb76da74bb07c" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" [[package]] name = "time-macros" -version = "0.2.22" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3526739392ec93fd8b359c8e98514cb3e8e021beb4e5f597b00a0221f8ed8a49" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" dependencies = [ "num-conv", "time-core", @@ -3643,9 +3641,9 @@ dependencies = [ [[package]] name = "tinystr" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d4f6d1145dcb577acf783d4e601bc1d76a13337bb54e6233add580b07344c8b" +checksum = "42d3e9c45c09de15d06dd8acf5f4e0e399e85927b7f00711024eb7ae10fa4869" dependencies = [ "displaydoc", "zerovec", @@ -3653,9 +3651,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" dependencies = [ "tinyvec_macros", ] @@ -3668,40 +3666,37 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.47.0" +version = "1.48.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43864ed400b6043a4757a25c7a64a8efde741aed79a056a2fb348a406701bb35" +checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408" dependencies = [ - "backtrace", "bytes", - "io-uring", "libc", "mio", "parking_lot", "pin-project-lite", "signal-hook-registry", - "slab", - "socket2 0.6.0", + "socket2 0.6.1", "tokio-macros", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "tokio-macros" -version = "2.5.0" +version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6e06d43f1345a3bcd39f6a56dbb7dcab2ba47e68e8ac134855e7e2bdbaf8cab8" +checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "tokio-rustls" -version = "0.26.2" +version = "0.26.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e727b36a1a0e8b74c376ac2211e40c2c8af09fb4013c60d910495810f008e9b" +checksum = "1729aa945f29d91ba541258c8df89027d5792d85a8841fb65e8bf0f4ede4ef61" dependencies = [ "rustls", "tokio", @@ -3709,9 +3704,9 @@ dependencies = [ [[package]] name = "tokio-util" -version = "0.7.15" +version = "0.7.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "66a539a9ad6d5d281510d5bd368c973d636c02dbf8a67300bfb6b950696ad7df" +checksum = "14307c986784f72ef81c89db7d9e28d6ac26d16213b109ea501696195e6e3ce5" dependencies = [ "bytes", "futures-core", @@ -3722,18 +3717,31 @@ dependencies = [ [[package]] name = "toml_datetime" -version = "0.6.11" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "22cddaf88f4fbc13c51aebbf5f8eceb5c7c5a9da2ac40a13519eb5b0a0e8f11c" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +dependencies = [ + "serde_core", +] [[package]] name = "toml_edit" -version = "0.22.27" +version = "0.23.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "41fe8c660ae4257887cf66394862d21dbca4a6ddd26f04a3560410406a2f819a" +checksum = "6485ef6d0d9b5d0ec17244ff7eb05310113c3f316f2d14200d4de56b3cb98f8d" dependencies = [ - "indexmap 2.10.0", + "indexmap 2.12.0", "toml_datetime", + "toml_parser", + "winnow", +] + +[[package]] +name = "toml_parser" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" +dependencies = [ "winnow", ] @@ -3801,7 +3809,7 @@ checksum = "81383ab64e72a7a8b8e13130c49e3dab29def6d0c7d76a03087b3cf71c5c6903" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -3827,14 +3835,14 @@ dependencies = [ [[package]] name = "tracing-subscriber" -version = "0.3.19" +version = "0.3.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8189decb5ac0fa7bc8b96b7cb9b2701d60d48805aca84a238004d665fcc4008" +checksum = "2054a14f5307d601f88daf0553e1cbf472acc4f2c51afab632431cdcd72124d5" dependencies = [ "matchers", "nu-ansi-term", "once_cell", - "regex", + "regex-automata", "sharded-slab", "smallvec", "thread_local", @@ -3851,18 +3859,9 @@ checksum = "e421abadd41a4225275504ea4d6566923418b7f05506fbc9c0fe86ba7396114b" [[package]] name = "twox-hash" -version = "2.1.1" +version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b907da542cbced5261bd3256de1b3a1bf340a3d37f93425a07362a1d687de56" - -[[package]] -name = "typed-builder" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a06fbd5b8de54c5f7c91f6fe4cebb949be2125d7758e630bb58b1d831dbce600" -dependencies = [ - "typed-builder-macro 0.19.1", -] +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" [[package]] name = "typed-builder" @@ -3870,18 +3869,7 @@ version = "0.20.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cd9d30e3a08026c78f246b173243cf07b3696d274debd26680773b6773c2afc7" dependencies = [ - "typed-builder-macro 0.20.1", -] - -[[package]] -name = "typed-builder-macro" -version = "0.19.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f9534daa9fd3ed0bd911d462a37f172228077e7abf18c18a5f67199d959205f8" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.104", + "typed-builder-macro", ] [[package]] @@ -3892,20 +3880,20 @@ checksum = "3c36781cc0e46a83726d9879608e4cf6c2505237e263a8eb8c24502989cfdb28" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "untrusted" @@ -3915,13 +3903,14 @@ checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" [[package]] name = "url" -version = "2.5.4" +version = "2.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "32f8b686cadd1473f4bd0117a5d28d36b1ade384ea9b5069a1c40aefed7fda60" +checksum = "08bc136a29a3d1758e07a9cca267be308aeebf5cfd5a10f3f67ab2097683ef5b" dependencies = [ "form_urlencoded", "idna", "percent-encoding", + "serde", ] [[package]] @@ -3932,11 +3921,11 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be" [[package]] name = "uuid" -version = "1.17.0" +version = "1.18.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d" +checksum = "2f87b8aa10b915a06587d0dec516c282ff295b475d94abf425d62b57710070a2" dependencies = [ - "getrandom 0.3.3", + "getrandom 0.3.4", "js-sys", "serde", "wasm-bindgen", @@ -3986,45 +3975,32 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "da95793dfc411fbbd93f5be7715b0578ec61fe87cb1a42b12eb625caa5c5ea60" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.104", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.55" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "551f88106c6d5e7ccc7cd9a16f312dd3b5d36ea8b4954304657d5dfba115d4a0" dependencies = [ "cfg-if", "js-sys", @@ -4035,9 +4011,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "04264334509e04a7bf8690f2384ef5265f05143a4bff3889ab7a3269adab59c2" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -4045,22 +4021,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "420bc339d9f322e562942d52e115d57e950d12d88983a14c79b86859ee6c7ebc" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.104", - "wasm-bindgen-backend", + "syn 2.0.108", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.105" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "76f218a38c84bcb33c25ec7059b07847d465ce0e0a76b995e134a45adcb6af76" dependencies = [ "unicode-ident", ] @@ -4080,9 +4056,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.82" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "3a1f95c0d03a47f4ae1f7a64643a6bb97465d9b740f0fa8f90ea33915c99a9a1" dependencies = [ "js-sys", "wasm-bindgen", @@ -4100,18 +4076,18 @@ dependencies = [ [[package]] name = "webpki-roots" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7e8983c3ab33d6fb807cfcdad2491c4ea8cbc8ed839181c7dfd9c67c83e261b2" +checksum = "32b130c0d2d49f8b6889abc456e795e82525204f27c42cf767cf0d7734e089b8" dependencies = [ "rustls-pki-types", ] [[package]] name = "widestring" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -4131,11 +4107,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -4144,33 +4120,11 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.61.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9babd3a767a4c1aef6900409f85f5d53ce2544ccdfaa86dad48c91782c6d6893" -dependencies = [ - "windows-collections", - "windows-core", - "windows-future", - "windows-link", - "windows-numerics", -] - -[[package]] -name = "windows-collections" -version = "0.2.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3beeceb5e5cfd9eb1d76b381630e82c4241ccd0d27f1a39ed41b2760b255c5e8" -dependencies = [ - "windows-core", -] - [[package]] name = "windows-core" -version = "0.61.2" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0fdd3ddb90610c7638aa2b3a3ab2904fb9e5cdbecc643ddb3647212781c4ae3" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", @@ -4179,69 +4133,48 @@ dependencies = [ "windows-strings", ] -[[package]] -name = "windows-future" -version = "0.2.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" -dependencies = [ - "windows-core", - "windows-link", - "windows-threading", -] - [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "windows-link" -version = "0.1.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5e6ad25900d524eaabdbbb96d20b4311e1e7ae1699af4fb28c17ae66c80d798a" - -[[package]] -name = "windows-numerics" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9150af68066c4c5c07ddc0ce30421554771e528bde427614c61038bc2c92c2b1" -dependencies = [ - "windows-core", - "windows-link", -] +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-result" -version = "0.3.4" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56f42bd332cc6c8eac5af113fc0c1fd6a8fd2aa08a0119358686e5160d0586c6" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.2" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "56e6c93f3a0c3b36176cb1327a4958a0353d5d166c2a35cb268ace15e91d3b57" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ "windows-link", ] @@ -4266,20 +4199,20 @@ dependencies = [ [[package]] name = "windows-sys" -version = "0.59.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e38bc4d79ed67fd075bcc251a1c39b32a1776bbe92e5bef1f0bf1f8c531853b" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" dependencies = [ - "windows-targets 0.52.6", + "windows-targets 0.53.5", ] [[package]] name = "windows-sys" -version = "0.60.2" +version = "0.61.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" dependencies = [ - "windows-targets 0.53.3", + "windows-link", ] [[package]] @@ -4315,28 +4248,19 @@ dependencies = [ [[package]] name = "windows-targets" -version = "0.53.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d5fe6031c4041849d7c496a8ded650796e7b6ecc19df1a431c1a363342e5dc91" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.0", - "windows_aarch64_msvc 0.53.0", - "windows_i686_gnu 0.53.0", - "windows_i686_gnullvm 0.53.0", - "windows_i686_msvc 0.53.0", - "windows_x86_64_gnu 0.53.0", - "windows_x86_64_gnullvm 0.53.0", - "windows_x86_64_msvc 0.53.0", -] - -[[package]] -name = "windows-threading" -version = "0.1.0" +version = "0.53.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b66463ad2e0ea3bbf808b7f1d371311c80e115c0b71d60efc142cafbcfb057a6" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" dependencies = [ "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", ] [[package]] @@ -4353,9 +4277,9 @@ checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" [[package]] name = "windows_aarch64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "86b8d5f90ddd19cb4a147a5fa63ca848db3df085e25fee3cc10b39b6eebae764" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" [[package]] name = "windows_aarch64_msvc" @@ -4371,9 +4295,9 @@ checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" [[package]] name = "windows_aarch64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c7651a1f62a11b8cbd5e0d42526e55f2c99886c77e007179efff86c2b137e66c" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" [[package]] name = "windows_i686_gnu" @@ -4389,9 +4313,9 @@ checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" [[package]] name = "windows_i686_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c1dc67659d35f387f5f6c479dc4e28f1d4bb90ddd1a5d3da2e5d97b42d6272c3" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" [[package]] name = "windows_i686_gnullvm" @@ -4401,9 +4325,9 @@ checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" [[package]] name = "windows_i686_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ce6ccbdedbf6d6354471319e781c0dfef054c81fbc7cf83f338a4296c0cae11" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" [[package]] name = "windows_i686_msvc" @@ -4419,9 +4343,9 @@ checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" [[package]] name = "windows_i686_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "581fee95406bb13382d2f65cd4a908ca7b1e4c2f1917f143ba16efe98a589b5d" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" [[package]] name = "windows_x86_64_gnu" @@ -4437,9 +4361,9 @@ checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" [[package]] name = "windows_x86_64_gnu" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e55b5ac9ea33f2fc1716d1742db15574fd6fc8dadc51caab1c16a3d3b4190ba" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" [[package]] name = "windows_x86_64_gnullvm" @@ -4455,9 +4379,9 @@ checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" [[package]] name = "windows_x86_64_gnullvm" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a6e035dd0599267ce1ee132e51c27dd29437f63325753051e71dd9e42406c57" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" [[package]] name = "windows_x86_64_msvc" @@ -4473,15 +4397,15 @@ checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] name = "windows_x86_64_msvc" -version = "0.53.0" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "271414315aff87387382ec3d271b52d7ae78726f5d44ac98b4f4030c91880486" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] name = "winnow" -version = "0.7.12" +version = "0.7.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3edebf492c8125044983378ecb5766203ad3b4c2f7a922bd7dd207f6d443e95" +checksum = "21a0236b59786fed61e2a80582dd500fe61f18b5dca67a4a067d0bc9039339cf" dependencies = [ "memchr", ] @@ -4497,19 +4421,16 @@ dependencies = [ ] [[package]] -name = "wit-bindgen-rt" -version = "0.39.0" +name = "wit-bindgen" +version = "0.46.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" -dependencies = [ - "bitflags", -] +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" [[package]] name = "writeable" -version = "0.6.1" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea2f10b9bb0928dfb1b42b65e1f9e36f7f54dbdf08457afefb38afcdec4fa2bb" +checksum = "9edde0db4769d2dc68579893f2306b26c6ecfbe0ef499b013d731b7b9247e0b9" [[package]] name = "wyz" @@ -4522,11 +4443,10 @@ dependencies = [ [[package]] name = "yoke" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f41bb01b8226ef4bfd589436a297c53d118f65921786300e427be8d487695cc" +checksum = "72d6e5c6afb84d73944e5cedb052c4680d5657337201555f9f2a16b7406d4954" dependencies = [ - "serde", "stable_deref_trait", "yoke-derive", "zerofrom", @@ -4534,34 +4454,34 @@ dependencies = [ [[package]] name = "yoke-derive" -version = "0.8.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "38da3c9736e16c5d3c8c597a9aaa5d1fa565d0532ae05e27c24aa62fb32c0ab6" +checksum = "b659052874eb698efe5b9e8cf382204678a0086ebf46982b79d6ca3182927e5d" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", "synstructure", ] [[package]] name = "zerocopy" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1039dd0d3c310cf05de012d8a39ff557cb0d23087fd44cad61df08fc31907a2f" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.26" +version = "0.8.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9ecf5b4cc5364572d7f4c329661bcc82724222973f2cab6f050a4e5c22f75181" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] @@ -4581,21 +4501,21 @@ checksum = "d71e5d6e06ab090c67b5e44993ec16b72dcbaabc526db883a360057678b48502" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", "synstructure", ] [[package]] name = "zeroize" -version = "1.8.1" +version = "1.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ced3678a2879b30306d323f4542626697a464a97c0a07c9aebf7ebca65cd4dde" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" [[package]] name = "zerotrie" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36f0bbd478583f79edad978b407914f61b2972f5af6fa089686016be8f9af595" +checksum = "2a59c17a5562d507e4b54960e8569ebee33bee890c70aa3fe7b97e85a9fd7851" dependencies = [ "displaydoc", "yoke", @@ -4604,9 +4524,9 @@ dependencies = [ [[package]] name = "zerovec" -version = "0.11.2" +version = "0.11.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4a05eb080e015ba39cc9e23bbe5e7fb04d5fb040350f99f34e338d5fdd294428" +checksum = "6c28719294829477f525be0186d13efa9a3c602f7ec202ca9e353d310fb9a002" dependencies = [ "yoke", "zerofrom", @@ -4615,20 +4535,20 @@ dependencies = [ [[package]] name = "zerovec-derive" -version = "0.11.1" +version = "0.11.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b96237efa0c878c64bd89c436f661be4e46b2f3eff1ebb976f7ef2321d2f58f" +checksum = "eadce39539ca5cb3985590102671f2567e659fca9666581ad3411d59207951f3" dependencies = [ "proc-macro2", "quote", - "syn 2.0.104", + "syn 2.0.108", ] [[package]] name = "zlib-rs" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "626bd9fa9734751fc50d6060752170984d7053f5a39061f524cda68023d4db8a" +checksum = "2f06ae92f42f5e5c42443fd094f245eb656abf56dd7cce9b8b263236565e00f2" [[package]] name = "zstd" @@ -4650,9 +4570,9 @@ dependencies = [ [[package]] name = "zstd-sys" -version = "2.0.15+zstd.1.5.7" +version = "2.0.16+zstd.1.5.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" dependencies = [ "cc", "pkg-config", diff --git a/iceberg_rust_ffi/Cargo.toml b/iceberg_rust_ffi/Cargo.toml index 13a822d..9eac1d8 100644 --- a/iceberg_rust_ffi/Cargo.toml +++ b/iceberg_rust_ffi/Cargo.toml @@ -12,14 +12,14 @@ default = ["julia"] julia = [] [dependencies] -iceberg = "0.6.0" +iceberg = { git = "https://github.com/RelationalAI/iceberg-rust.git", rev = "ccb1e0c9983a1bdcb5b70fa637759df526a9a75e" } object_store_ffi = { git = "https://github.com/RelationalAI/object_store_ffi", rev = "79b08071c7a1642532b5891253280861eca9e44e", default-features = false } tokio = { version = "1.0", features = ["full"] } futures = "0.3" libc = "0.2" anyhow = "1.0" -arrow-array = "55.2.0" -arrow-ipc = "55.2.0" +arrow-array = "56.2.0" +arrow-ipc = "56.2.0" tracing-subscriber = "0.3" tracing = "0.1" once_cell = "1.19" diff --git a/iceberg_rust_ffi/src/full.rs b/iceberg_rust_ffi/src/full.rs new file mode 100644 index 0000000..6da7486 --- /dev/null +++ b/iceberg_rust_ffi/src/full.rs @@ -0,0 +1,80 @@ +use std::ffi::{c_char, c_void, CStr}; +use std::ptr; + +use iceberg::scan::{TableScan, TableScanBuilder}; +use object_store_ffi::{ + export_runtime_op, with_cancellation, CResult, NotifyGuard, ResponseGuard, RT, +}; +use tokio::sync::Mutex as AsyncMutex; + +use crate::scan_common::*; +use crate::{IcebergArrowStream, IcebergArrowStreamResponse, IcebergTable}; + +/// Struct for regular (full) scan builder and scan +#[repr(C)] +pub struct IcebergScan { + pub builder: Option>, + pub scan: Option, +} + +unsafe impl Send for IcebergScan {} + +/// Create a new scan builder +#[no_mangle] +pub extern "C" fn iceberg_new_scan(table: *mut IcebergTable) -> *mut IcebergScan { + if table.is_null() { + return ptr::null_mut(); + } + let table_ref = unsafe { &*table }; + let scan_builder = table_ref.table.scan(); + Box::into_raw(Box::new(IcebergScan { + builder: Some(scan_builder), + scan: None, + })) +} + +// Use macros from scan_common for shared functionality +impl_select_columns!(iceberg_select_columns, IcebergScan); + +impl_scan_builder_method!( + iceberg_scan_with_data_file_concurrency_limit, + IcebergScan, + with_data_file_concurrency_limit +); + +impl_scan_builder_method!( + iceberg_scan_with_manifest_entry_concurrency_limit, + IcebergScan, + with_manifest_entry_concurrency_limit +); + +impl_with_batch_size!(iceberg_scan_with_batch_size, IcebergScan); + +impl_scan_build!(iceberg_scan_build, IcebergScan); + +// Async function to initialize stream from a table scan +export_runtime_op!( + iceberg_arrow_stream, + IcebergArrowStreamResponse, + || { + if scan.is_null() { + return Err(anyhow::anyhow!("Null scan pointer provided")); + } + let scan_ref = unsafe { &(*scan).scan }; + if scan_ref.is_none() { + return Err(anyhow::anyhow!("Scan not initialized")); + } + + Ok(scan_ref.as_ref().unwrap()) + }, + scan_ref, + async { + let stream = scan_ref.to_arrow().await?; + Ok::(IcebergArrowStream { + stream: AsyncMutex::new(stream), + }) + }, + scan: *mut IcebergScan +); + +impl_scan_free!(iceberg_scan_free, IcebergScan); diff --git a/iceberg_rust_ffi/src/incremental.rs b/iceberg_rust_ffi/src/incremental.rs new file mode 100644 index 0000000..18e10ce --- /dev/null +++ b/iceberg_rust_ffi/src/incremental.rs @@ -0,0 +1,137 @@ +use std::ffi::{c_char, c_void, CStr}; +use std::ptr; + +use iceberg::scan::incremental::{IncrementalTableScan, IncrementalTableScanBuilder}; +use object_store_ffi::{ + export_runtime_op, with_cancellation, CResult, Context, NotifyGuard, RawResponse, + ResponseGuard, RT, +}; +use tokio::sync::Mutex as AsyncMutex; + +use crate::scan_common::*; +use crate::{IcebergArrowStream, IcebergTable}; + +/// Struct for incremental scan builder and scan +#[repr(C)] +pub struct IcebergIncrementalScan { + pub builder: Option>, + pub scan: Option, +} + +unsafe impl Send for IcebergIncrementalScan {} + +/// Response type for unzipped streams (two separate Arrow streams) +#[repr(C)] +pub struct IcebergUnzippedStreamsResponse { + result: CResult, + inserts_stream: *mut IcebergArrowStream, + deletes_stream: *mut IcebergArrowStream, + error_message: *mut c_char, + context: *const Context, +} + +unsafe impl Send for IcebergUnzippedStreamsResponse {} + +impl RawResponse for IcebergUnzippedStreamsResponse { + type Payload = (IcebergArrowStream, IcebergArrowStream); + + fn result_mut(&mut self) -> &mut CResult { + &mut self.result + } + + fn context_mut(&mut self) -> &mut *const Context { + &mut self.context + } + + fn error_message_mut(&mut self) -> &mut *mut c_char { + &mut self.error_message + } + + fn set_payload(&mut self, payload: Option) { + match payload { + Some((inserts, deletes)) => { + self.inserts_stream = Box::into_raw(Box::new(inserts)); + self.deletes_stream = Box::into_raw(Box::new(deletes)); + } + None => { + self.inserts_stream = ptr::null_mut(); + self.deletes_stream = ptr::null_mut(); + } + } + } +} + +/// Create a new incremental scan builder +#[no_mangle] +pub extern "C" fn iceberg_new_incremental_scan( + table: *mut IcebergTable, + from_snapshot_id: i64, + to_snapshot_id: i64, +) -> *mut IcebergIncrementalScan { + if table.is_null() { + return ptr::null_mut(); + } + let table_ref = unsafe { &*table }; + let scan_builder = table_ref + .table + .incremental_scan(from_snapshot_id, to_snapshot_id); + Box::into_raw(Box::new(IcebergIncrementalScan { + builder: Some(scan_builder), + scan: None, + })) +} + +// Use macros from scan_common for shared functionality +impl_select_columns!(iceberg_incremental_select_columns, IcebergIncrementalScan); + +impl_scan_builder_method!( + iceberg_incremental_scan_with_data_file_concurrency_limit, + IcebergIncrementalScan, + with_concurrency_limit_data_files +); + +impl_scan_builder_method!( + iceberg_incremental_scan_with_manifest_entry_concurrency_limit, + IcebergIncrementalScan, + with_concurrency_limit_manifest_entries +); + +impl_with_batch_size!(iceberg_incremental_scan_with_batch_size, IcebergIncrementalScan); + +impl_scan_build!(iceberg_incremental_scan_build, IcebergIncrementalScan); + +// Get unzipped Arrow streams from incremental scan (async) +// Returns two separate streams: one for inserts, one for deletes +export_runtime_op!( + iceberg_incremental_arrow_stream_unzipped, + IcebergUnzippedStreamsResponse, + || { + if scan.is_null() { + return Err(anyhow::anyhow!("Null scan pointer provided")); + } + let scan_ref = unsafe { &(*scan).scan }; + if scan_ref.is_none() { + return Err(anyhow::anyhow!("Incremental scan not initialized")); + } + + Ok(scan_ref.as_ref().unwrap()) + }, + scan_ref, + async { + // Get unzipped streams (separate append and delete streams) + let (inserts_stream, deletes_stream) = scan_ref.to_unzipped_arrow().await?; + + let inserts = IcebergArrowStream { + stream: AsyncMutex::new(inserts_stream), + }; + + let deletes = IcebergArrowStream { + stream: AsyncMutex::new(deletes_stream), + }; + + Ok::<(IcebergArrowStream, IcebergArrowStream), anyhow::Error>((inserts, deletes)) + }, + scan: *mut IcebergIncrementalScan +); + +impl_scan_free!(iceberg_free_incremental_scan, IcebergIncrementalScan); diff --git a/iceberg_rust_ffi/src/lib.rs b/iceberg_rust_ffi/src/lib.rs index 5ded550..56787ee 100644 --- a/iceberg_rust_ffi/src/lib.rs +++ b/iceberg_rust_ffi/src/lib.rs @@ -7,7 +7,6 @@ use anyhow::Result; use arrow_array::RecordBatch; use arrow_ipc::writer::StreamWriter; use iceberg::io::FileIOBuilder; -use iceberg::scan::{TableScan, TableScanBuilder}; use iceberg::table::{StaticTable, Table}; use iceberg::TableIdent; @@ -18,6 +17,15 @@ use object_store_ffi::{ ResultCallback, RESULT_CB, RT, }; +// Modules for scan functionality +mod scan_common; +mod full; +mod incremental; + +// Re-export scan types and functions +pub use full::IcebergScan; +pub use incremental::{IcebergIncrementalScan, IcebergUnzippedStreamsResponse}; + // We use `jl_adopt_thread` to ensure Rust can call into Julia when notifying // the Base.Event that is waiting for the Rust result. // Note that this will be linked in from the Julia process, we do not try @@ -80,13 +88,6 @@ pub struct IcebergTable { pub table: Table, } -#[repr(C)] -pub struct IcebergScan { - pub builder: Option>, - pub scan: Option, -} - -unsafe impl Send for IcebergScan {} // Stream wrapper for FFI - using async mutex to avoid blocking calls #[repr(C)] @@ -320,175 +321,6 @@ export_runtime_op!( snapshot_path: *const c_char ); -#[no_mangle] -pub extern "C" fn iceberg_new_scan(table: *mut IcebergTable) -> *mut IcebergScan { - if table.is_null() { - return ptr::null_mut(); - } - let table_ref = unsafe { &*table }; - let scan_builder = table_ref.table.scan(); - return Box::into_raw(Box::new(IcebergScan { - builder: Some(scan_builder), - scan: None, - })); -} - -#[no_mangle] -pub extern "C" fn iceberg_select_columns( - scan: &mut *mut IcebergScan, - column_names: *const *const c_char, - num_columns: usize, -) -> CResult { - if scan.is_null() || (*scan).is_null() || column_names.is_null() { - return CResult::Error; - } - - let mut columns = Vec::new(); - - for i in 0..num_columns { - let col_ptr = unsafe { *column_names.add(i) }; - if col_ptr.is_null() { - return CResult::Error; - } - - let col_str = unsafe { - match CStr::from_ptr(col_ptr).to_str() { - Ok(s) => s, - Err(_) => return CResult::Error, - } - }; - columns.push(col_str.to_string()); - } - - let scan_ref = unsafe { Box::from_raw(*scan) }; - - if scan_ref.builder.is_none() { - return CResult::Error; - } - *scan = Box::into_raw(Box::new(IcebergScan { - builder: scan_ref.builder.map(|b| b.select(columns)), - scan: scan_ref.scan, - })); - - return CResult::Ok; -} - -#[no_mangle] -pub extern "C" fn iceberg_scan_with_data_file_concurrency_limit( - scan: &mut *mut IcebergScan, - n: usize, -) -> CResult { - if scan.is_null() || (*scan).is_null() { - return CResult::Error; - } - let scan_ref = unsafe { Box::from_raw(*scan) }; - - if scan_ref.builder.is_none() { - return CResult::Error; - } - - *scan = Box::into_raw(Box::new(IcebergScan { - builder: scan_ref - .builder - .map(|b| b.with_data_file_concurrency_limit(n)), - scan: scan_ref.scan, - })); - - return CResult::Ok; -} - -#[no_mangle] -pub extern "C" fn iceberg_scan_with_manifest_entry_concurrency_limit( - scan: &mut *mut IcebergScan, - n: usize, -) -> CResult { - if scan.is_null() || (*scan).is_null() { - return CResult::Error; - } - let scan_ref = unsafe { Box::from_raw(*scan) }; - - if scan_ref.builder.is_none() { - return CResult::Error; - } - - *scan = Box::into_raw(Box::new(IcebergScan { - builder: scan_ref - .builder - .map(|b| b.with_manifest_entry_concurrency_limit(n)), - scan: scan_ref.scan, - })); - - return CResult::Ok; -} - -#[no_mangle] -pub extern "C" fn iceberg_scan_with_batch_size(scan: &mut *mut IcebergScan, n: usize) -> CResult { - if scan.is_null() || (*scan).is_null() { - return CResult::Error; - } - let scan_ref = unsafe { Box::from_raw(*scan) }; - - if scan_ref.builder.is_none() { - return CResult::Error; - } - - assert!(scan_ref.scan.is_none()); - - *scan = Box::into_raw(Box::new(IcebergScan { - builder: scan_ref.builder.map(|b| b.with_batch_size(Some(n))), - scan: None, - })); - - return CResult::Ok; -} - -#[no_mangle] -pub extern "C" fn iceberg_scan_build(scan: &mut *mut IcebergScan) -> CResult { - if scan.is_null() || (*scan).is_null() { - return CResult::Error; - } - let scan_ref = unsafe { Box::from_raw(*scan) }; - if scan_ref.builder.is_none() { - return CResult::Error; - } - let builder = scan_ref.builder.unwrap(); - - match builder.build() { - Ok(table_scan) => { - *scan = Box::into_raw(Box::new(IcebergScan { - builder: None, - scan: Some(table_scan), - })); - CResult::Ok - } - Err(_) => CResult::Error, - } -} - -// Async function to initialize stream from a table scan without getting first batch -export_runtime_op!( - iceberg_arrow_stream, - IcebergArrowStreamResponse, - || { - if scan.is_null() { - return Err(anyhow::anyhow!("Null scan pointer provided")); - } - let scan_ref = unsafe { &(*scan).scan }; - if scan_ref.is_none() { - return Err(anyhow::anyhow!("Scan not initialized")); - } - - return Ok(scan_ref.as_ref().unwrap()); - }, - scan_ref, - async { - let stream = scan_ref.to_arrow().await?; - Ok::(IcebergArrowStream { - stream: AsyncMutex::new(stream), - }) - }, - scan: *mut IcebergScan -); // Async function to get next batch from existing stream export_runtime_op!( @@ -530,16 +362,6 @@ pub extern "C" fn iceberg_table_free(table: *mut IcebergTable) { } } -#[no_mangle] -pub extern "C" fn iceberg_scan_free(scan: &mut *mut IcebergScan) { - if !scan.is_null() { - unsafe { - let _ = Box::from_raw(*scan); - *scan = ptr::null_mut(); - } - } -} - #[no_mangle] pub extern "C" fn iceberg_arrow_stream_free(stream: *mut IcebergArrowStream) { if !stream.is_null() { diff --git a/iceberg_rust_ffi/src/scan_common.rs b/iceberg_rust_ffi/src/scan_common.rs new file mode 100644 index 0000000..7d59b72 --- /dev/null +++ b/iceberg_rust_ffi/src/scan_common.rs @@ -0,0 +1,145 @@ +/// Common macros for scan builder methods shared between regular and incremental scans +/// Macro to generate select_columns function for any scan type +macro_rules! impl_select_columns { + ($fn_name:ident, $scan_type:ident) => { + #[no_mangle] + pub extern "C" fn $fn_name( + scan: &mut *mut $scan_type, + column_names: *const *const c_char, + num_columns: usize, + ) -> CResult { + if scan.is_null() || (*scan).is_null() || column_names.is_null() { + return CResult::Error; + } + + let mut columns = Vec::new(); + + for i in 0..num_columns { + let col_ptr = unsafe { *column_names.add(i) }; + if col_ptr.is_null() { + return CResult::Error; + } + + let col_str = unsafe { + match CStr::from_ptr(col_ptr).to_str() { + Ok(s) => s, + Err(_) => return CResult::Error, + } + }; + columns.push(col_str.to_string()); + } + + let scan_ref = unsafe { Box::from_raw(*scan) }; + + if scan_ref.builder.is_none() { + return CResult::Error; + } + *scan = Box::into_raw(Box::new($scan_type { + builder: scan_ref.builder.map(|b| b.select(columns)), + scan: scan_ref.scan, + })); + + CResult::Ok + } + }; +} + +/// Macro to generate scan builder methods that take a usize parameter +macro_rules! impl_scan_builder_method { + ($fn_name:ident, $scan_type:ident, $builder_method:ident) => { + #[no_mangle] + pub extern "C" fn $fn_name(scan: &mut *mut $scan_type, n: usize) -> CResult { + if scan.is_null() || (*scan).is_null() { + return CResult::Error; + } + let scan_ref = unsafe { Box::from_raw(*scan) }; + + if scan_ref.builder.is_none() { + return CResult::Error; + } + + *scan = Box::into_raw(Box::new($scan_type { + builder: scan_ref.builder.map(|b| b.$builder_method(n)), + scan: scan_ref.scan, + })); + + CResult::Ok + } + }; +} + +/// Macro to generate with_batch_size function for any scan type +macro_rules! impl_with_batch_size { + ($fn_name:ident, $scan_type:ident) => { + #[no_mangle] + pub extern "C" fn $fn_name(scan: &mut *mut $scan_type, n: usize) -> CResult { + if scan.is_null() || (*scan).is_null() { + return CResult::Error; + } + let scan_ref = unsafe { Box::from_raw(*scan) }; + + if scan_ref.builder.is_none() { + return CResult::Error; + } + + assert!(scan_ref.scan.is_none()); + + *scan = Box::into_raw(Box::new($scan_type { + builder: scan_ref.builder.map(|b| b.with_batch_size(Some(n))), + scan: None, + })); + + CResult::Ok + } + }; +} + +/// Macro to generate build function for any scan type +macro_rules! impl_scan_build { + ($fn_name:ident, $scan_type:ident) => { + #[no_mangle] + pub extern "C" fn $fn_name(scan: &mut *mut $scan_type) -> CResult { + if scan.is_null() || (*scan).is_null() { + return CResult::Error; + } + let scan_ref = unsafe { Box::from_raw(*scan) }; + if scan_ref.builder.is_none() { + return CResult::Error; + } + let builder = scan_ref.builder.unwrap(); + + match builder.build() { + Ok(built_scan) => { + *scan = Box::into_raw(Box::new($scan_type { + builder: None, + scan: Some(built_scan), + })); + CResult::Ok + } + Err(_) => CResult::Error, + } + } + }; +} + +/// Macro to generate scan_free function for any scan type +macro_rules! impl_scan_free { + ($fn_name:ident, $scan_type:ident) => { + #[no_mangle] + pub extern "C" fn $fn_name(scan: &mut *mut $scan_type) { + if !scan.is_null() { + unsafe { + let _ = Box::from_raw(*scan); + *scan = std::ptr::null_mut(); + } + } + } + }; +} + +// Re-export macros for use in other modules +pub(crate) use impl_scan_builder_method; +pub(crate) use impl_scan_build; +pub(crate) use impl_scan_free; +pub(crate) use impl_select_columns; +pub(crate) use impl_with_batch_size; diff --git a/src/RustyIceberg.jl b/src/RustyIceberg.jl index 18e3c1f..f34d66c 100644 --- a/src/RustyIceberg.jl +++ b/src/RustyIceberg.jl @@ -7,10 +7,13 @@ using Libdl using Arrow using iceberg_rust_ffi_jll -export Table, Scan, ArrowBatch, StaticConfig, ArrowStream -export TableIterator, TableIteratorState -export init_runtime, read_table +export Table, Scan, IncrementalScan, ArrowBatch, StaticConfig, ArrowStream +export init_runtime export IcebergException +export new_incremental_scan, free_incremental_scan! +export table_open, free_table, new_scan, free_scan! +export select_columns!, with_batch_size!, with_data_file_concurrency_limit!, with_manifest_entry_concurrency_limit! +export scan!, next_batch, free_batch, free_stream const Option{T} = Union{T, Nothing} @@ -166,20 +169,49 @@ function wait_or_cancel(event::Base.Event, response) end end -# Opaque pointer types +""" + Table + +Opaque pointer type representing an Iceberg table handle from the Rust FFI layer. + +Create a table using `table_open` and free it with `free_table` when done. +""" const Table = Ptr{Cvoid} -const Scan = Ptr{Cvoid} -const ScanRef = Ref{Scan} -const ArrowStream = Ptr{Cvoid} -# Arrow batch structure +""" + ArrowBatch + +Structure representing a batch of Arrow data from the Rust FFI layer. + +# Fields +- `data::Ptr{UInt8}`: Pointer to the Arrow IPC format data +- `length::Csize_t`: Length of the data buffer in bytes +- `rust_ptr::Ptr{Cvoid}`: Rust-side pointer for memory management + +Batches should be freed using `free_batch` after processing. +""" struct ArrowBatch data::Ptr{UInt8} length::Csize_t rust_ptr::Ptr{Cvoid} end -# Response structures for async operations +# Include scan modules +include("scan_common.jl") +include("full.jl") +include("incremental.jl") + +""" + TableResponse + +Response structure for asynchronous table operations. + +# Fields +- `result::Cint`: Result code from the operation (0 for success) +- `table::Table`: The opened table handle +- `error_message::Ptr{Cchar}`: Error message string if operation failed +- `context::Ptr{Cvoid}`: Context pointer for operation cancellation +""" mutable struct TableResponse result::Cint table::Table @@ -189,33 +221,33 @@ mutable struct TableResponse TableResponse() = new(-1, C_NULL, C_NULL, C_NULL) end -mutable struct Response - result::Cint - error_message::Ptr{Cchar} - context::Ptr{Cvoid} +""" + Response - Response() = new(-1, C_NULL, C_NULL) -end +Generic response structure for asynchronous operations. -mutable struct ArrowStreamResponse +# Fields +- `result::Cint`: Result code from the operation (0 for success) +- `error_message::Ptr{Cchar}`: Error message string if operation failed +- `context::Ptr{Cvoid}`: Context pointer for operation cancellation +""" +mutable struct Response result::Cint - stream::ArrowStream error_message::Ptr{Cchar} context::Ptr{Cvoid} - ArrowStreamResponse() = new(-1, C_NULL, C_NULL, C_NULL) + Response() = new(-1, C_NULL, C_NULL) end -mutable struct BatchResponse - result::Cint - batch::Ptr{ArrowBatch} - error_message::Ptr{Cchar} - context::Ptr{Cvoid} +""" + IcebergException <: Exception - BatchResponse() = new(-1, C_NULL, C_NULL, C_NULL) -end +Exception type for Iceberg operations. -# Exception types +# Fields +- `msg::String`: Error message describing what went wrong +- `code::Union{Int,Nothing}`: Optional error code from the FFI layer +""" struct IcebergException <: Exception msg::String code::Union{Int,Nothing} @@ -259,164 +291,7 @@ function table_open(snapshot_path::String) end """ - new_scan(table::Table) -> IcebergScan - -Create a scan for the given table. -""" -function new_scan(table::Table) - scan = @ccall rust_lib.iceberg_new_scan(table::Table)::Ptr{Cvoid} - return Ref(scan) -end - -""" - select_columns!(scan::ScanRef, column_names::Vector{String})::Cint - -Select specific columns for the scan. -""" -function select_columns!(scan::ScanRef, column_names::Vector{String}) - # Convert String vector to Cstring array - c_strings = [pointer(col) for col in column_names] - result = @ccall rust_lib.iceberg_select_columns( - convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, - pointer(c_strings)::Ptr{Cstring}, - length(column_names)::Csize_t - )::Cint - - if result != 0 - error("Failed to select columns") - end - return nothing -end - -""" - with_data_file_concurrency_limit!(scan::ScanRef, n::UInt)::Cint - -Sets the data file concurrency level for the scan. -""" -function with_data_file_concurrency_limit!(scan::ScanRef, n::UInt) - return @ccall rust_lib.iceberg_scan_with_data_file_concurrency( - convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, - n::Csize_t - )::Cint -end - -""" - with_manifest_entry_concurrency_limit!(scan::ScanRef, n::UInt)::Cint - -Sets the manifest entry concurrency level for the scan. -""" -function with_manifest_entry_concurrency_limit!(scan::ScanRef, n::UInt) - return @ccall rust_lib.iceberg_scan_with_manifest_entry_concurrency( - convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, - n::Csize_t - )::Cint -end - -""" - with_batch_size!(scan::ScanRef, n::UInt)::Cint - -Sets the batch size for the scan. -""" -function with_batch_size!(scan::ScanRef, n::UInt) - return @ccall rust_lib.iceberg_scan_with_batch_size( - convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, - n::Csize_t - )::Cint -end - -""" - build!(scan::ScanRef)::Cint - -Build the provided table scan object. -""" -function build!(scan::ScanRef) - return _build!(convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))) -end - -function _build!(scan::Ptr{Ptr{Cvoid}}) - return @ccall rust_lib.iceberg_scan_build(scan::Ptr{Ptr{Cvoid}})::Cint -end - -""" - scan!(scan::ScanRef) -> Cint - -Build the provided table scan object. -""" -function scan!(scan::ScanRef) - result = build!(scan) - if result != 0 - throw(IcebergException("Failed to build scan", result)) - end - - return arrow_stream(scan[]) -end - -""" - arrow_stream(scan::Scan)::IcebergArrowStream - -Initialize an Arrow stream for the scan asynchronously. -""" -function arrow_stream(scan::Scan) - response = ArrowStreamResponse() - ct = current_task() - event = Base.Event() - handle = pointer_from_objref(event) - - preserve_task(ct) - result = GC.@preserve response event try - result = @ccall rust_lib.iceberg_arrow_stream( - scan::Scan, - response::Ref{ArrowStreamResponse}, - handle::Ptr{Cvoid} - )::Cint - - wait_or_cancel(event, response) - - result - finally - unpreserve_task(ct) - end - - @throw_on_error(response, "iceberg_arrow_stream", IcebergException) - - return response.stream -end - -""" - next_batch(scan::Scan)::Ptr{ArrowBatch} - -Wait for the next batch from the initialized stream asynchronously and return it directly. -Returns C_NULL if end of stream is reached. -""" -function next_batch(stream::ArrowStream) - response = BatchResponse() - ct = current_task() - event = Base.Event() - handle = pointer_from_objref(event) - - preserve_task(ct) - result = GC.@preserve response event try - result = @ccall rust_lib.iceberg_next_batch( - stream::ArrowStream, - response::Ref{BatchResponse}, - handle::Ptr{Cvoid} - )::Cint - - wait_or_cancel(event, response) - - result - finally - unpreserve_task(ct) - end - - @throw_on_error(response, "iceberg_next_batch", IcebergException) - - # Return the batch pointer directly - return response.batch -end - -""" - free_table(table::IcebergTable) + free_table(table::Table) Free the memory associated with an Iceberg table. """ @@ -424,215 +299,4 @@ function free_table(table::Table) @ccall rust_lib.iceberg_table_free(table::Table)::Cvoid end -""" - free_batch(batch::Ptr{ArrowBatch}) - -Free the memory associated with an Arrow batch. -""" -function free_batch(batch::Ptr{ArrowBatch}) - @ccall rust_lib.iceberg_arrow_batch_free(batch::Ptr{ArrowBatch})::Cvoid -end - -""" - free_stream(stream::ArrowStream) - -Free the memory associated with an Arrow stream. -""" -function free_stream(stream::ArrowStream) - @ccall rust_lib.iceberg_arrow_stream_free(stream::ArrowStream)::Cvoid -end - -""" - free_scan!(scan::IcebergScanRef) - -Free the memory associated with a scan. -""" -function free_scan!(scan::ScanRef) - @ccall rust_lib.iceberg_scan_free( - convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}} - )::Cvoid -end - - -# Iterator type for Arrow batches -struct TableIterator - snapshot_path::String - columns::Vector{String} - batch_size::Union{UInt,Nothing} - data_file_concurrency_limit::Union{UInt,Nothing} - manifest_entry_concurrency_limit::Union{UInt,Nothing} -end - -# Iterator state -mutable struct TableIteratorState - table::Table - scan::Ref{Ptr{Cvoid}} - stream::ArrowStream - is_open::Bool - batch_ptr::Ptr{ArrowBatch} - - function TableIteratorState(table, scan, stream, is_open) - state = new(table, scan, stream, is_open, C_NULL) - # Ensure cleanup happens even if iterator is abandoned - finalizer(_cleanup_iterator_state!, state) - return state - end -end - -function _cleanup_iterator_state!(state::TableIteratorState) - if state.is_open - try - # Only free batch if we know we have one pending to prevent double-free - state.batch_ptr != C_NULL && free_batch(state.batch_ptr) - @assert state.stream != C_NULL - @assert state.scan != C_NULL - @assert state.table != C_NULL - free_stream(state.stream) - free_scan!(state.scan) - free_table(state.table) - catch e - # Log but don't throw in finalizer - # TODO: should we keep this log or throw? - @error "Error in IcebergTableIteratorState finalizer: $(e)" - finally - state.is_open = false - state.batch_ptr = C_NULL - state.stream = C_NULL - state.scan = C_NULL - state.table = C_NULL - end - end -end - -""" - Base.iterate(iter::IcebergTableIterator, state=nothing) - -Iterate over `Arrow.Table` objects from the Iceberg table. -""" -function Base.iterate(iter::TableIterator, state=nothing) - local arrow_table - local should_cleanup_resources = false - - try - if isnothing(state) - # First iteration - ensure runtime is initialized - if !iceberg_started() - init_runtime() - end - - table, scan, stream = nothing, nothing, nothing - - try - # Open table - table = table_open(iter.snapshot_path) - - # Create scan - scan = new_scan(table) - - # Select columns if specified - if !isempty(iter.columns) - select_columns!(scan, iter.columns) - end - - if !isnothing(iter.data_file_concurrency_limit) - with_data_file_concurrency_limit!(scan, iter.data_file_concurrency_limit) - end - - if !isnothing(iter.manifest_entry_concurrency_limit) - with_manifest_entry_concurrency_limit!(scan, iter.manifest_entry_concurrency_limit) - end - - if !isnothing(iter.batch_size) - with_batch_size!(scan, iter.batch_size) - end - - stream = scan!(scan) - state = TableIteratorState(table, scan, stream, true) - catch e - !isnothing(stream) && free_stream(stream) - !isnothing(scan) && free_scan!(scan) - !isnothing(table) && free_table(table) - rethrow(e) - end - else - # Clean up the batch from the previous iteration. - if state.batch_ptr != C_NULL - free_batch(state.batch_ptr) - state.batch_ptr = C_NULL - end - end - - # Wait for next batch asynchronously - state.batch_ptr = next_batch(state.stream) - - if state.batch_ptr == C_NULL - # End of stream - mark for cleanup and return nothing - should_cleanup_resources = true - return nothing - end - - # Convert ArrowBatch pointer to ArrowBatch struct - batch = unsafe_load(state.batch_ptr) - - # Read Arrow data - arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) - - return arrow_table, state - - catch e - # On exception, mark resources for cleanup before rethrowing - should_cleanup_resources = true - rethrow(e) - finally - # Clean up scan and table resources only if needed (end of stream or exception) - if should_cleanup_resources && state !== nothing && state.is_open - state.batch_ptr != C_NULL && free_batch(state.batch_ptr) - free_stream(state.stream) - free_scan!(state.scan) - free_table(state.table) - state.stream = C_NULL - state.scan = C_NULL - state.table = C_NULL - # Mark as closed to prevent finalizer from double-freeing - state.is_open = false - end - end -end - -""" - Base.eltype(::Type{IcebergTableIterator}) - -Return the element type of the iterator. -""" -Base.eltype(::Type{TableIterator}) = Arrow.Table - -""" - Base.IteratorSize(::Type{IcebergTableIterator}) - -Return the size trait of the iterator. -""" -Base.IteratorSize(::Type{TableIterator}) = Base.SizeUnknown() - -# High-level Julia interface -""" - read_table(table_path::String, metadata_path::String; columns::Vector{String}=String[]) -> IcebergTableIterator - -Read an Iceberg table and return an iterator over Arrow.Table objects. -""" -function read_table( - snapshot_path::String; - columns::Vector{String}=String[], - batch_size::Union{UInt, Nothing}=nothing, - data_file_concurrency_limit::Union{UInt, Nothing}=nothing, - manifest_entry_concurrency_limit::Union{UInt, Nothing}=nothing -) - return TableIterator( - snapshot_path, - columns, - batch_size, - data_file_concurrency_limit, - manifest_entry_concurrency_limit - ) -end - end # module RustyIceberg diff --git a/src/full.jl b/src/full.jl new file mode 100644 index 0000000..4f8ceec --- /dev/null +++ b/src/full.jl @@ -0,0 +1,196 @@ +# Regular (full) table scan implementation + +""" + Scan + +A mutable wrapper around a pointer to a regular (full) table scan. + +This type enables multiple dispatch and safe memory management for Iceberg table scans. +Use `new_scan` to create a scan, configure it with builder methods, and call `scan!` +to build the scan and obtain an Arrow stream. + +# Example +```julia +table = table_open("s3://path/to/table/metadata.json") +scan = new_scan(table) +select_columns!(scan, ["col1", "col2"]) +with_batch_size!(scan, UInt(1024)) +stream = scan!(scan) +# ... process batches from stream +free_stream(stream) +free_scan!(scan) +free_table(table) +``` +""" +mutable struct Scan + ptr::Ptr{Cvoid} +end + +""" + new_scan(table::Table) -> Scan + +Create a scan for the given table. +""" +function new_scan(table::Table) + scan_ptr = @ccall rust_lib.iceberg_new_scan(table::Table)::Ptr{Cvoid} + return Scan(scan_ptr) +end + +""" + select_columns!(scan::Scan, column_names::Vector{String}) + +Select specific columns for the scan. +""" +function select_columns!(scan::Scan, column_names::Vector{String}) + # Convert String vector to Cstring array + c_strings = [pointer(col) for col in column_names] + result = @ccall rust_lib.iceberg_select_columns( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + pointer(c_strings)::Ptr{Cstring}, + length(column_names)::Csize_t + )::Cint + + if result != 0 + error("Failed to select columns") + end + return nothing +end + +""" + with_data_file_concurrency_limit!(scan::Scan, n::UInt) + +Sets the data file concurrency level for the scan. +""" +function with_data_file_concurrency_limit!(scan::Scan, n::UInt) + result = @ccall rust_lib.iceberg_scan_with_data_file_concurrency_limit( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + n::Csize_t + )::Cint + + if result != 0 + error("Failed to set data file concurrency limit") + end + return nothing +end + +""" + with_manifest_entry_concurrency_limit!(scan::Scan, n::UInt) + +Sets the manifest entry concurrency level for the scan. +""" +function with_manifest_entry_concurrency_limit!(scan::Scan, n::UInt) + result = @ccall rust_lib.iceberg_scan_with_manifest_entry_concurrency_limit( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + n::Csize_t + )::Cint + + if result != 0 + error("Failed to set manifest entry concurrency limit") + end + return nothing +end + +""" + with_batch_size!(scan::Scan, n::UInt) + +Sets the batch size for the scan. +""" +function with_batch_size!(scan::Scan, n::UInt) + result = @ccall rust_lib.iceberg_scan_with_batch_size( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + n::Csize_t + )::Cint + + if result != 0 + error("Failed to set batch size") + end + return nothing +end + +""" + build!(scan::Scan) + +Build the provided table scan object. +""" +function build!(scan::Scan) + result = @ccall rust_lib.iceberg_scan_build( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}} + )::Cint + + if result != 0 + throw(IcebergException("Failed to build scan", result)) + end + return nothing +end + +""" + ArrowStreamResponse + +Response structure for asynchronous Arrow stream initialization operations. + +# Fields +- `result::Cint`: Result code from the operation (0 for success) +- `stream::ArrowStream`: The initialized Arrow stream +- `error_message::Ptr{Cchar}`: Error message string if operation failed +- `context::Ptr{Cvoid}`: Context pointer for operation cancellation +""" +mutable struct ArrowStreamResponse + result::Cint + stream::ArrowStream + error_message::Ptr{Cchar} + context::Ptr{Cvoid} + + ArrowStreamResponse() = new(-1, C_NULL, C_NULL, C_NULL) +end + +""" + arrow_stream(scan::Scan)::ArrowStream + +Initialize an Arrow stream for the scan asynchronously. +""" +function arrow_stream(scan::Scan) + response = ArrowStreamResponse() + ct = current_task() + event = Base.Event() + handle = pointer_from_objref(event) + + preserve_task(ct) + result = GC.@preserve response event try + result = @ccall rust_lib.iceberg_arrow_stream( + scan.ptr::Ptr{Cvoid}, + response::Ref{ArrowStreamResponse}, + handle::Ptr{Cvoid} + )::Cint + + wait_or_cancel(event, response) + + result + finally + unpreserve_task(ct) + end + + @throw_on_error(response, "iceberg_arrow_stream", IcebergException) + + return response.stream +end + +""" + scan!(scan::Scan) -> ArrowStream + +Build the provided table scan object and return an Arrow stream. +""" +function scan!(scan::Scan) + build!(scan) + return arrow_stream(scan) +end + +""" + free_scan!(scan::Scan) + +Free the memory associated with a scan. +""" +function free_scan!(scan::Scan) + @ccall rust_lib.iceberg_scan_free( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}} + )::Cvoid +end diff --git a/src/incremental.jl b/src/incremental.jl new file mode 100644 index 0000000..245dd2a --- /dev/null +++ b/src/incremental.jl @@ -0,0 +1,204 @@ +# Incremental table scan implementation + +""" + IncrementalScan + +A mutable wrapper around a pointer to an incremental table scan. + +Incremental scans read changes between two snapshots in an Iceberg table. +Use `new_incremental_scan` to create a scan between two snapshot IDs, configure it +with builder methods, and call `scan!` to obtain separate Arrow streams for inserts and deletes. + +# Example +```julia +table = table_open("s3://path/to/table/metadata.json") +scan = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) +with_batch_size!(scan, UInt(1024)) +inserts_stream, deletes_stream = scan!(scan) +# ... process batches from both streams +free_stream(inserts_stream) +free_stream(deletes_stream) +free_incremental_scan!(scan) +free_table(table) +``` +""" +mutable struct IncrementalScan + ptr::Ptr{Cvoid} +end + +""" + UnzippedStreamsResponse + +Response structure for asynchronous incremental scan operations that return separate streams. + +# Fields +- `result::Cint`: Result code from the operation (0 for success) +- `inserts_stream::ArrowStream`: Stream containing inserted rows +- `deletes_stream::ArrowStream`: Stream containing position delete metadata +- `error_message::Ptr{Cchar}`: Error message string if operation failed +- `context::Ptr{Cvoid}`: Context pointer for operation cancellation +""" +mutable struct UnzippedStreamsResponse + result::Cint + inserts_stream::ArrowStream + deletes_stream::ArrowStream + error_message::Ptr{Cchar} + context::Ptr{Cvoid} + + UnzippedStreamsResponse() = new(-1, C_NULL, C_NULL, C_NULL, C_NULL) +end + +""" + new_incremental_scan(table::Table, from_snapshot_id::Int64, to_snapshot_id::Int64) -> IncrementalScan + +Create an incremental scan for the given table between two snapshots. +""" +function new_incremental_scan(table::Table, from_snapshot_id::Int64, to_snapshot_id::Int64) + scan_ptr = @ccall rust_lib.iceberg_new_incremental_scan( + table::Table, + from_snapshot_id::Int64, + to_snapshot_id::Int64 + )::Ptr{Cvoid} + return IncrementalScan(scan_ptr) +end + +""" + select_columns!(scan::IncrementalScan, column_names::Vector{String}) + +Select specific columns for the incremental scan. +""" +function select_columns!(scan::IncrementalScan, column_names::Vector{String}) + # Convert String vector to Cstring array + c_strings = [pointer(col) for col in column_names] + result = @ccall rust_lib.iceberg_incremental_select_columns( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + pointer(c_strings)::Ptr{Cstring}, + length(column_names)::Csize_t + )::Cint + + if result != 0 + error("Failed to select columns for incremental scan") + end + return nothing +end + +""" + with_data_file_concurrency_limit!(scan::IncrementalScan, n::UInt) + +Sets the data file concurrency level for the incremental scan. +""" +function with_data_file_concurrency_limit!(scan::IncrementalScan, n::UInt) + result = @ccall rust_lib.iceberg_incremental_scan_with_data_file_concurrency_limit( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + n::Csize_t + )::Cint + + if result != 0 + error("Failed to set data file concurrency limit for incremental scan") + end + return nothing +end + +""" + with_manifest_entry_concurrency_limit!(scan::IncrementalScan, n::UInt) + +Sets the manifest entry concurrency level for the incremental scan. +""" +function with_manifest_entry_concurrency_limit!(scan::IncrementalScan, n::UInt) + result = @ccall rust_lib.iceberg_incremental_scan_with_manifest_entry_concurrency_limit( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + n::Csize_t + )::Cint + + if result != 0 + error("Failed to set manifest entry concurrency limit for incremental scan") + end + return nothing +end + +""" + with_batch_size!(scan::IncrementalScan, n::UInt) + +Sets the batch size for the incremental scan. +""" +function with_batch_size!(scan::IncrementalScan, n::UInt) + result = @ccall rust_lib.iceberg_incremental_scan_with_batch_size( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}}, + n::Csize_t + )::Cint + + if result != 0 + error("Failed to set batch size for incremental scan") + end + return nothing +end + +""" + build!(scan::IncrementalScan) + +Build the provided incremental table scan object. +""" +function build!(scan::IncrementalScan) + result = @ccall rust_lib.iceberg_incremental_scan_build( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}} + )::Cint + + if result != 0 + throw(IcebergException("Failed to build incremental scan", result)) + end + return nothing +end + +""" + incremental_arrow_stream_unzipped(scan::IncrementalScan) -> (ArrowStream, ArrowStream) + +Initialize unzipped Arrow streams (inserts and deletes) for the incremental scan asynchronously. +Returns a tuple of (inserts_stream, deletes_stream). +""" +function incremental_arrow_stream_unzipped(scan::IncrementalScan) + response = UnzippedStreamsResponse() + ct = current_task() + event = Base.Event() + handle = pointer_from_objref(event) + + preserve_task(ct) + result = GC.@preserve response event try + result = @ccall rust_lib.iceberg_incremental_arrow_stream_unzipped( + scan.ptr::Ptr{Cvoid}, + response::Ref{UnzippedStreamsResponse}, + handle::Ptr{Cvoid} + )::Cint + + wait_or_cancel(event, response) + + result + finally + unpreserve_task(ct) + end + + @throw_on_error(response, "iceberg_incremental_arrow_stream_unzipped", IcebergException) + + return (response.inserts_stream, response.deletes_stream) +end + +""" + scan!(scan::IncrementalScan) -> (ArrowStream, ArrowStream) + +Build the provided incremental table scan object and return unzipped Arrow streams. +Returns a tuple of (inserts_stream, deletes_stream). +""" +function scan!(scan::IncrementalScan) + build!(scan) + return incremental_arrow_stream_unzipped(scan) +end + +""" + free_incremental_scan!(scan::IncrementalScan) + +Free the memory associated with an incremental scan. +""" +function free_incremental_scan!(scan::IncrementalScan) + @ccall rust_lib.iceberg_free_incremental_scan( + convert(Ptr{Ptr{Cvoid}}, pointer_from_objref(scan))::Ptr{Ptr{Cvoid}} + )::Cvoid +end diff --git a/src/scan_common.jl b/src/scan_common.jl new file mode 100644 index 0000000..bd8775a --- /dev/null +++ b/src/scan_common.jl @@ -0,0 +1,80 @@ +# Common scan utilities shared between full and incremental scans + +""" + ArrowStream + +Opaque pointer type representing an Arrow stream from the Rust FFI layer. +This stream can be used to fetch batches of Arrow data asynchronously. +""" +const ArrowStream = Ptr{Cvoid} + +""" + BatchResponse + +Response structure for asynchronous batch operations. + +# Fields +- `result::Cint`: Result code from the operation (0 for success) +- `batch::Ptr{ArrowBatch}`: Pointer to the Arrow batch data +- `error_message::Ptr{Cchar}`: Error message string if operation failed +- `context::Ptr{Cvoid}`: Context pointer for operation cancellation +""" +mutable struct BatchResponse + result::Cint + batch::Ptr{ArrowBatch} + error_message::Ptr{Cchar} + context::Ptr{Cvoid} + + BatchResponse() = new(-1, C_NULL, C_NULL, C_NULL) +end + +""" + next_batch(stream::ArrowStream)::Ptr{ArrowBatch} + +Wait for the next batch from the initialized stream asynchronously and return it directly. +Returns C_NULL if end of stream is reached. +""" +function next_batch(stream::ArrowStream) + response = BatchResponse() + ct = current_task() + event = Base.Event() + handle = pointer_from_objref(event) + + preserve_task(ct) + result = GC.@preserve response event try + result = @ccall rust_lib.iceberg_next_batch( + stream::ArrowStream, + response::Ref{BatchResponse}, + handle::Ptr{Cvoid} + )::Cint + + wait_or_cancel(event, response) + + result + finally + unpreserve_task(ct) + end + + @throw_on_error(response, "iceberg_next_batch", IcebergException) + + # Return the batch pointer directly + return response.batch +end + +""" + free_batch(batch::Ptr{ArrowBatch}) + +Free the memory associated with an Arrow batch. +""" +function free_batch(batch::Ptr{ArrowBatch}) + @ccall rust_lib.iceberg_arrow_batch_free(batch::Ptr{ArrowBatch})::Cvoid +end + +""" + free_stream(stream::ArrowStream) + +Free the memory associated with an Arrow stream. +""" +function free_stream(stream::ArrowStream) + @ccall rust_lib.iceberg_arrow_stream_free(stream::ArrowStream)::Cvoid +end diff --git a/test/runtests.jl b/test/runtests.jl index 8703f75..de7292b 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -14,52 +14,81 @@ using Arrow println("✅ Runtime initialization successful") end -@testset "High-level API" begin +@testset "Low-level API" begin # Test with the actual customer table that we know works snapshot_path = "s3://warehouse/tpch.sf01/customer/metadata/00001-76f6e7e4-b34f-492f-b6a1-cc9f8c8f4975.metadata.json" - println("Testing high-level API...") + println("Testing low-level API...") println(" Snapshot path: $snapshot_path") - # Test creating table iterator - table_iterator = read_table(snapshot_path) - @test table_iterator isa TableIterator - println("✅ Table iterator created successfully") + # Open table + table = RustyIceberg.table_open(snapshot_path) + @test table != C_NULL + println("✅ Table opened successfully") - # Test iteration over Arrow.Table objects + # Create scan + scan = RustyIceberg.new_scan(table) + @test scan isa RustyIceberg.Scan + @test scan.ptr != C_NULL + println("✅ Scan created successfully") + + # Build and get stream + stream = RustyIceberg.scan!(scan) + @test stream != C_NULL + println("✅ Stream obtained successfully") + + # Test iteration over batches arrow_tables = Arrow.Table[] batch_count = 0 total_rows = 0 - for arrow_table in table_iterator - batch_count += 1 - push!(arrow_tables, arrow_table) + try + while true + batch_ptr = RustyIceberg.next_batch(stream) + if batch_ptr == C_NULL + break + end - # Convert to DataFrame for testing - df = DataFrame(arrow_table) - @test !isempty(df) - total_rows += nrow(df) + batch_count += 1 + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + @test arrow_table isa Arrow.Table + push!(arrow_tables, arrow_table) - # Only print details for first few batches to avoid spam - if batch_count <= 3 - println("📦 Batch $batch_count: $(size(df)) rows × $(length(names(df))) columns") - println(" → Columns: $(names(df))") - end + # Convert to DataFrame for testing + df = DataFrame(arrow_table) + @test !isempty(df) + total_rows += nrow(df) - # Stop after a few batches for testing to avoid long test times - if batch_count >= 5 - println(" ... stopping after $batch_count batches for testing") - break + # Only print details for first few batches to avoid spam + if batch_count <= 3 + println("📦 Batch $batch_count: $(size(df)) rows × $(length(names(df))) columns") + println(" → Columns: $(names(df))") + end + + RustyIceberg.free_batch(batch_ptr) + + # Stop after a few batches for testing to avoid long test times + if batch_count >= 5 + println(" ... stopping after $batch_count batches for testing") + break + end end - end - @test batch_count > 0 - @test total_rows > 0 - @test !isempty(arrow_tables) - println("✅ High-level API iteration test successful") - println(" - Total batches processed: $batch_count") - println(" - Total rows processed: $total_rows") - println(" - Total Arrow tables: $(length(arrow_tables))") + @test batch_count > 0 + @test total_rows > 0 + @test !isempty(arrow_tables) + println("✅ Low-level API iteration test successful") + println(" - Total batches processed: $batch_count") + println(" - Total rows processed: $total_rows") + println(" - Total Arrow tables: $(length(arrow_tables))") + finally + # Clean up + RustyIceberg.free_stream(stream) + RustyIceberg.free_scan!(scan) + RustyIceberg.free_table(table) + println("✅ Resources cleaned up") + end # Test reading with column selection if !isempty(arrow_tables) @@ -68,61 +97,49 @@ end if !isempty(names(first_df)) # Select first two columns for testing selected_columns = names(first_df)[1:min(2, length(names(first_df)))] - selected_iterator = read_table( - snapshot_path; columns=selected_columns, batch_size=UInt(8) - ) - @test selected_iterator isa TableIterator - - selected_arrow_tables = Arrow.Table[] - selected_batch_count = 0 - for arrow_table in selected_iterator - @test length(arrow_table) <= 8 - selected_batch_count += 1 - push!(selected_arrow_tables, arrow_table) - - # Only process first batch for column selection test - if selected_batch_count >= 1 - break + + table2 = RustyIceberg.table_open(snapshot_path) + scan2 = RustyIceberg.new_scan(table2) + RustyIceberg.select_columns!(scan2, selected_columns) + RustyIceberg.with_batch_size!(scan2, UInt(8)) + stream2 = RustyIceberg.scan!(scan2) + + try + selected_arrow_tables = Arrow.Table[] + selected_batch_count = 0 + + batch_ptr = RustyIceberg.next_batch(stream2) + if batch_ptr != C_NULL + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + @test arrow_table isa Arrow.Table + @test length(arrow_table) <= 8 + push!(selected_arrow_tables, arrow_table) + RustyIceberg.free_batch(batch_ptr) end - end - @test !isempty(selected_arrow_tables) + @test !isempty(selected_arrow_tables) - # Check that selected columns match - if !isempty(selected_arrow_tables) + # Check that selected columns match selected_df = DataFrame(selected_arrow_tables[1]) @test names(selected_df) == selected_columns println("✅ Column selection test successful") println(" - Selected columns: $(names(selected_df))") + finally + # Clean up + RustyIceberg.free_stream(stream2) + RustyIceberg.free_scan!(scan2) + RustyIceberg.free_table(table2) end end end end -@testset "Iterator Properties" begin - # Test iterator type properties - snapshot_path = "s3://warehouse/tpch.sf01/customer/metadata/00001-76f6e7e4-b34f-492f-b6a1-cc9f8c8f4975.metadata.json" - - table_iterator = read_table(snapshot_path) - - # Test eltype - @test Base.eltype(table_iterator) == Arrow.Table - - # Test IteratorSize - @test Base.IteratorSize(table_iterator) == Base.SizeUnknown() - - println("✅ Iterator properties test successful") -end - @testset "Error Handling" begin # Test with invalid paths - this should throw an exception in our async API try - invalid_iterator = read_table("invalid/path", "invalid/metadata.json") - # Try to iterate - this should fail - for arrow_table in invalid_iterator - @test false # Should not reach here - break - end + table = RustyIceberg.table_open("invalid/path/metadata.json") + @test false # Should not reach here catch e @test e isa Exception println("✅ Error handling test successful: caught expected exception") @@ -134,51 +151,518 @@ end nations_snapshot_path = "s3://warehouse/tpch.sf01/nation/metadata/00001-44f668fe-3688-49d5-851f-36e75d143321.metadata.json" println("Testing reading nations table...") - nations_iterator = read_table(nations_snapshot_path; batch_size=UInt(5)) - @test nations_iterator isa TableIterator + + table = RustyIceberg.table_open(nations_snapshot_path) + scan = RustyIceberg.new_scan(table) + RustyIceberg.with_batch_size!(scan, UInt(5)) + stream = RustyIceberg.scan!(scan) rows = Tuple[] - for arrow_table in nations_iterator - df = DataFrame(arrow_table) - expected_columns = ["n_nationkey", "n_name", "n_regionkey", "n_comment"] - @test names(df) == expected_columns - # Collect rows as tuples for easier verification - for row in eachrow(df) - push!(rows, Tuple(row)) - end - end - - @test rows == Tuple[ - (0, "ALGERIA", 0, "furiously regular requests. platelets affix furious"), - (1, "ARGENTINA", 1, "instructions wake quickly. final deposits haggle. final, silent theodolites "), - (2, "BRAZIL", 1, "asymptotes use fluffily quickly bold instructions. slyly bold dependencies sleep carefully pending accounts"), - (3, "CANADA", 1, "ss deposits wake across the pending foxes. packages after the carefully bold requests integrate caref"), - (4, "EGYPT", 4, "usly ironic, pending foxes. even, special instructions nag. sly, final foxes detect slyly fluffily "), - (5, "ETHIOPIA", 0, "regular requests sleep carefull"), - (6, "FRANCE", 3, "oggedly. regular packages solve across"), - (7, "GERMANY", 3, "ong the regular requests: blithely silent pinto beans hagg"), - (8, "INDIA", 2, "uriously unusual deposits about the slyly final pinto beans could"), - (9, "INDONESIA", 2, "d deposits sleep quickly according to the dogged, regular dolphins. special excuses haggle furiously special reque"), - (10, "IRAN", 4, "furiously idle platelets nag. express asymptotes s"), - (11, "IRAQ", 4, "pendencies; slyly express foxes integrate carefully across the reg"), - (12, "JAPAN", 2, " quickly final packages. furiously i"), - (13, "JORDAN", 4, "the slyly regular ideas. silent Tiresias affix slyly fu"), - (14, "KENYA", 0, "lyly special foxes. slyly regular deposits sleep carefully. carefully permanent accounts slee"), - (15, "MOROCCO", 0, "ct blithely: blithely express accounts nag carefully. silent packages haggle carefully abo"), - (16, "MOZAMBIQUE", 0, " beans after the carefully regular accounts r"), - (17, "PERU", 1, "ly final foxes. blithely ironic accounts haggle. regular foxes about the regular deposits are furiously ir"), - (18, "CHINA", 2, "ckly special packages cajole slyly. unusual, unusual theodolites mold furiously. slyly sile"), - (19, "ROMANIA", 3, "sly blithe requests. thinly bold deposits above the blithely regular accounts nag special, final requests. care"), - (20, "SAUDI ARABIA", 4, "se slyly across the blithely regular deposits. deposits use carefully regular "), - (21, "VIETNAM", 2, "lly across the quickly even pinto beans. caref"), - (22, "RUSSIA", 3, "uctions. furiously unusual instructions sleep furiously ironic packages. slyly "), - (23, "UNITED KINGDOM", 3, "carefully pending courts sleep above the ironic, regular theo"), - (24, "UNITED STATES", 1, "ly ironic requests along the slyly bold ideas hang after the blithely special notornis; blithely even accounts") - ] + batch_ptr = C_NULL + try + while true + batch_ptr = RustyIceberg.next_batch(stream) + if batch_ptr == C_NULL + break + end + + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + df = DataFrame(arrow_table) + expected_columns = ["n_nationkey", "n_name", "n_regionkey", "n_comment"] + @test names(df) == expected_columns + # Collect rows as tuples for easier verification + for row in eachrow(df) + push!(rows, Tuple(row)) + end + + RustyIceberg.free_batch(batch_ptr) + batch_ptr = C_NULL + end + @test rows == Tuple[ + (0, "ALGERIA", 0, "furiously regular requests. platelets affix furious"), + (1, "ARGENTINA", 1, "instructions wake quickly. final deposits haggle. final, silent theodolites "), + (2, "BRAZIL", 1, "asymptotes use fluffily quickly bold instructions. slyly bold dependencies sleep carefully pending accounts"), + (3, "CANADA", 1, "ss deposits wake across the pending foxes. packages after the carefully bold requests integrate caref"), + (4, "EGYPT", 4, "usly ironic, pending foxes. even, special instructions nag. sly, final foxes detect slyly fluffily "), + (5, "ETHIOPIA", 0, "regular requests sleep carefull"), + (6, "FRANCE", 3, "oggedly. regular packages solve across"), + (7, "GERMANY", 3, "ong the regular requests: blithely silent pinto beans hagg"), + (8, "INDIA", 2, "uriously unusual deposits about the slyly final pinto beans could"), + (9, "INDONESIA", 2, "d deposits sleep quickly according to the dogged, regular dolphins. special excuses haggle furiously special reque"), + (10, "IRAN", 4, "furiously idle platelets nag. express asymptotes s"), + (11, "IRAQ", 4, "pendencies; slyly express foxes integrate carefully across the reg"), + (12, "JAPAN", 2, " quickly final packages. furiously i"), + (13, "JORDAN", 4, "the slyly regular ideas. silent Tiresias affix slyly fu"), + (14, "KENYA", 0, "lyly special foxes. slyly regular deposits sleep carefully. carefully permanent accounts slee"), + (15, "MOROCCO", 0, "ct blithely: blithely express accounts nag carefully. silent packages haggle carefully abo"), + (16, "MOZAMBIQUE", 0, " beans after the carefully regular accounts r"), + (17, "PERU", 1, "ly final foxes. blithely ironic accounts haggle. regular foxes about the regular deposits are furiously ir"), + (18, "CHINA", 2, "ckly special packages cajole slyly. unusual, unusual theodolites mold furiously. slyly sile"), + (19, "ROMANIA", 3, "sly blithe requests. thinly bold deposits above the blithely regular accounts nag special, final requests. care"), + (20, "SAUDI ARABIA", 4, "se slyly across the blithely regular deposits. deposits use carefully regular "), + (21, "VIETNAM", 2, "lly across the quickly even pinto beans. caref"), + (22, "RUSSIA", 3, "uctions. furiously unusual instructions sleep furiously ironic packages. slyly "), + (23, "UNITED KINGDOM", 3, "carefully pending courts sleep above the ironic, regular theo"), + (24, "UNITED STATES", 1, "ly ironic requests along the slyly bold ideas hang after the blithely special notornis; blithely even accounts") + ] + finally + if batch_ptr != C_NULL + RustyIceberg.free_batch(batch_ptr) + end + RustyIceberg.free_stream(stream) + RustyIceberg.free_scan!(scan) + RustyIceberg.free_table(table) + end println("✅ Nations table read and verified successfully") end +@testset "Incremental Scan API" begin + println("Testing incremental scan API...") + + # Use the test table created specifically for incremental scan testing + test_snapshot_path = "s3://warehouse/incremental/test1/metadata/00003-359e8bb8-1e5d-46d2-bcde-fdaeaa41114f.metadata.json" + + # Open the table + table = RustyIceberg.table_open(test_snapshot_path) + @test table != C_NULL + println("✅ Table opened successfully") + + # Use real snapshot IDs from the test table + from_snapshot_id = Int64(6540713100348352610) + to_snapshot_id = Int64(6832180054960511692) + + # The table is created with these transactions. Above snapshot IDs are for the 2nd and the 4th txn below: + #= + CREATE TABLE demo.incremental.test1 USING iceberg + TBLPROPERTIES ('write.delete.mode' = 'merge-on-read') + AS (SELECT n FROM range(1, 11) r(n)); + + INSERT INTO incremental.test1 + SELECT n FROM range(101, 200) r(n); + + INSERT INTO incremental.test1 + SELECT n FROM range(201, 300) r(n); + + DELETE FROM incremental.test1 WHERE n = 150 OR n = 250; + =# + + @testset "Incremental Scan E2E Test" begin + scan = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) + @test scan isa RustyIceberg.IncrementalScan + @test scan.ptr != C_NULL + println("✅ Incremental scan created (from snapshot $from_snapshot_id to $to_snapshot_id)") + + # Test builder methods + @test_nowarn RustyIceberg.with_batch_size!(scan, UInt(50)) + println("✅ Batch size configured") + + # Build and get streams + inserts_stream, deletes_stream = RustyIceberg.scan!(scan) + @test inserts_stream != C_NULL + @test deletes_stream != C_NULL + println("✅ Streams obtained successfully") + + try + # Read and validate from both streams + inserts_values = Int64[] + deletes_values = Int64[] + inserts_batches = 0 + deletes_batches = 0 + + # Read from inserts stream + while true + batch_ptr = RustyIceberg.next_batch(inserts_stream) + if batch_ptr == C_NULL + break + end + inserts_batches += 1 + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + @test arrow_table isa Arrow.Table + + # Convert to DataFrame and collect values from column "n" + df = DataFrame(arrow_table) + @test "n" in names(df) + append!(inserts_values, df.n) + + RustyIceberg.free_batch(batch_ptr) + end + + # Read from deletes stream + # Position deletes return metadata (pos, file_path) not actual row data + deletes_values = Tuple{String, Int64}[] + while true + batch_ptr = RustyIceberg.next_batch(deletes_stream) + if batch_ptr == C_NULL + break + end + deletes_batches += 1 + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + @test arrow_table isa Arrow.Table + + # Convert to DataFrame and extract position delete metadata + df = DataFrame(arrow_table) + for row in eachrow(df) + push!(deletes_values, (row.file_path, row.pos)) + end + + RustyIceberg.free_batch(batch_ptr) + end + + println("✅ Successfully read from incremental scan streams") + println(" - Inserts batches: $inserts_batches") + println(" - Deletes batches: $deletes_batches") + println(" - Inserts rows: $(length(inserts_values))") + println(" - Delete records: $(length(deletes_values))") + + # Validate deletes: should have 1 delete record for row 150 + # (row 250 was added and deleted in the same incremental range, so it's filtered out) + @test length(deletes_values) == 1 + + # Sort by position for consistent ordering + sort!(deletes_values, by = x -> x[2]) + + # Extract positions and file paths + positions = [x[2] for x in deletes_values] + file_paths = [x[1] for x in deletes_values] + + # Validate the position values and file paths + @test positions == [10] # position for row 150 in corresponding data file + @test all(endswith.(file_paths, ".parquet")) + + # Validate inserts: should have n from 201 to 299 inclusive, except 250 + # That's 98 rows: 201-249 (49) + 251-299 (49) + @test length(inserts_values) == 98 + + # Sort for easier validation + sort!(inserts_values) + + # Check range and missing 250 + @test minimum(inserts_values) == 201 + @test maximum(inserts_values) == 299 + @test 250 ∉ inserts_values + + # Verify exact expected set + expected_inserts = vcat(201:249, 251:299) + @test inserts_values == expected_inserts + println("✅ Inserts validated: n from 201-299 (excluding 250), total $(length(inserts_values)) rows") + finally + # Clean up + RustyIceberg.free_stream(inserts_stream) + RustyIceberg.free_stream(deletes_stream) + RustyIceberg.free_incremental_scan!(scan) + println("✅ Resources cleaned up") + end + end + + @testset "Incremental Scan with Column Selection" begin + scan2 = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) + + # Select only the "n" column + RustyIceberg.select_columns!(scan2, ["n"]) + + inserts_stream2, deletes_stream2 = RustyIceberg.scan!(scan2) + + try + # Read one batch from inserts to verify column selection + batch_ptr = RustyIceberg.next_batch(inserts_stream2) + if batch_ptr != C_NULL + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + df = DataFrame(arrow_table) + + # Should only have the "n" column + @test names(df) == ["n"] + @test !isempty(df) + + RustyIceberg.free_batch(batch_ptr) + println("✅ Incremental scan column selection test successful") + end + finally + RustyIceberg.free_stream(inserts_stream2) + RustyIceberg.free_stream(deletes_stream2) + RustyIceberg.free_incremental_scan!(scan2) + end + end + + # Clean up table + RustyIceberg.free_table(table) + println("✅ Incremental scan test completed successfully!") +end + +@testset "Builder API Tests" begin + println("Testing builder API methods...") + + customer_path = "s3://warehouse/tpch.sf01/customer/metadata/00001-76f6e7e4-b34f-492f-b6a1-cc9f8c8f4975.metadata.json" + incremental_path = "s3://warehouse/incremental/test1/metadata/00003-359e8bb8-1e5d-46d2-bcde-fdaeaa41114f.metadata.json" + from_snapshot_id = Int64(6540713100348352610) + to_snapshot_id = Int64(6832180054960511692) + + @testset "select_columns! - Full Scan" begin + table = RustyIceberg.table_open(customer_path) + scan = RustyIceberg.new_scan(table) + + # Select specific columns + RustyIceberg.select_columns!(scan, ["c_custkey", "c_name"]) + stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(stream) + @test batch_ptr != C_NULL + + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + df = DataFrame(arrow_table) + + @test names(df) == ["c_custkey", "c_name"] + @test !isempty(df) + + RustyIceberg.free_batch(batch_ptr) + println("✅ select_columns! test passed for full scan") + finally + RustyIceberg.free_stream(stream) + RustyIceberg.free_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "select_columns! - Incremental Scan" begin + table = RustyIceberg.table_open(incremental_path) + scan = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) + + RustyIceberg.select_columns!(scan, ["n"]) + inserts_stream, deletes_stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(inserts_stream) + if batch_ptr != C_NULL + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + df = DataFrame(arrow_table) + + @test names(df) == ["n"] + @test !isempty(df) + + RustyIceberg.free_batch(batch_ptr) + end + println("✅ select_columns! test passed for incremental scan") + finally + RustyIceberg.free_stream(inserts_stream) + RustyIceberg.free_stream(deletes_stream) + RustyIceberg.free_incremental_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "with_batch_size! - Full Scan" begin + table = RustyIceberg.table_open(customer_path) + scan = RustyIceberg.new_scan(table) + + # Set small batch size + RustyIceberg.with_batch_size!(scan, UInt(10)) + stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(stream) + @test batch_ptr != C_NULL + + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + + # Batch size should be respected (at most 10 rows) + @test length(arrow_table) <= 10 + + RustyIceberg.free_batch(batch_ptr) + println("✅ with_batch_size! test passed for full scan") + finally + RustyIceberg.free_stream(stream) + RustyIceberg.free_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "with_batch_size! - Incremental Scan" begin + table = RustyIceberg.table_open(incremental_path) + scan = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) + + RustyIceberg.with_batch_size!(scan, UInt(10)) + inserts_stream, deletes_stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(inserts_stream) + if batch_ptr != C_NULL + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + + @test length(arrow_table) <= 10 + + RustyIceberg.free_batch(batch_ptr) + end + println("✅ with_batch_size! test passed for incremental scan") + finally + RustyIceberg.free_stream(inserts_stream) + RustyIceberg.free_stream(deletes_stream) + RustyIceberg.free_incremental_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "with_data_file_concurrency_limit! - Full Scan" begin + table = RustyIceberg.table_open(customer_path) + scan = RustyIceberg.new_scan(table) + + # Set concurrency limit (should not error) + @test_nowarn RustyIceberg.with_data_file_concurrency_limit!(scan, UInt(4)) + stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(stream) + @test batch_ptr != C_NULL + + RustyIceberg.free_batch(batch_ptr) + println("✅ with_data_file_concurrency_limit! test passed for full scan") + finally + RustyIceberg.free_stream(stream) + RustyIceberg.free_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "with_data_file_concurrency_limit! - Incremental Scan" begin + table = RustyIceberg.table_open(incremental_path) + scan = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) + + @test_nowarn RustyIceberg.with_data_file_concurrency_limit!(scan, UInt(4)) + inserts_stream, deletes_stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(inserts_stream) + if batch_ptr != C_NULL + RustyIceberg.free_batch(batch_ptr) + end + println("✅ with_data_file_concurrency_limit! test passed for incremental scan") + finally + RustyIceberg.free_stream(inserts_stream) + RustyIceberg.free_stream(deletes_stream) + RustyIceberg.free_incremental_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "with_manifest_entry_concurrency_limit! - Full Scan" begin + table = RustyIceberg.table_open(customer_path) + scan = RustyIceberg.new_scan(table) + + # Set concurrency limit (should not error) + @test_nowarn RustyIceberg.with_manifest_entry_concurrency_limit!(scan, UInt(4)) + stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(stream) + @test batch_ptr != C_NULL + + RustyIceberg.free_batch(batch_ptr) + println("✅ with_manifest_entry_concurrency_limit! test passed for full scan") + finally + RustyIceberg.free_stream(stream) + RustyIceberg.free_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "with_manifest_entry_concurrency_limit! - Incremental Scan" begin + table = RustyIceberg.table_open(incremental_path) + scan = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) + + @test_nowarn RustyIceberg.with_manifest_entry_concurrency_limit!(scan, UInt(4)) + inserts_stream, deletes_stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(inserts_stream) + if batch_ptr != C_NULL + RustyIceberg.free_batch(batch_ptr) + end + println("✅ with_manifest_entry_concurrency_limit! test passed for incremental scan") + finally + RustyIceberg.free_stream(inserts_stream) + RustyIceberg.free_stream(deletes_stream) + RustyIceberg.free_incremental_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "Combined Builder Methods - Full Scan" begin + table = RustyIceberg.table_open(customer_path) + scan = RustyIceberg.new_scan(table) + + # Combine multiple builder methods + RustyIceberg.select_columns!(scan, ["c_custkey", "c_name", "c_address"]) + RustyIceberg.with_batch_size!(scan, UInt(5)) + RustyIceberg.with_data_file_concurrency_limit!(scan, UInt(2)) + RustyIceberg.with_manifest_entry_concurrency_limit!(scan, UInt(2)) + + stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(stream) + @test batch_ptr != C_NULL + + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + df = DataFrame(arrow_table) + + # Verify all configurations + @test names(df) == ["c_custkey", "c_name", "c_address"] + @test length(arrow_table) <= 5 + @test !isempty(df) + + RustyIceberg.free_batch(batch_ptr) + println("✅ Combined builder methods test passed for full scan") + finally + RustyIceberg.free_stream(stream) + RustyIceberg.free_scan!(scan) + RustyIceberg.free_table(table) + end + end + + @testset "Combined Builder Methods - Incremental Scan" begin + table = RustyIceberg.table_open(incremental_path) + scan = new_incremental_scan(table, from_snapshot_id, to_snapshot_id) + + # Combine multiple builder methods + RustyIceberg.select_columns!(scan, ["n"]) + RustyIceberg.with_batch_size!(scan, UInt(5)) + RustyIceberg.with_data_file_concurrency_limit!(scan, UInt(2)) + RustyIceberg.with_manifest_entry_concurrency_limit!(scan, UInt(2)) + + inserts_stream, deletes_stream = RustyIceberg.scan!(scan) + + try + batch_ptr = RustyIceberg.next_batch(inserts_stream) + if batch_ptr != C_NULL + batch = unsafe_load(batch_ptr) + arrow_table = Arrow.Table(unsafe_wrap(Array, batch.data, batch.length)) + df = DataFrame(arrow_table) + + @test names(df) == ["n"] + @test length(arrow_table) <= 5 + @test !isempty(df) + + RustyIceberg.free_batch(batch_ptr) + end + println("✅ Combined builder methods test passed for incremental scan") + finally + RustyIceberg.free_stream(inserts_stream) + RustyIceberg.free_stream(deletes_stream) + RustyIceberg.free_incremental_scan!(scan) + RustyIceberg.free_table(table) + end + end +end + end # End of testset println("\n🎉 All tests completed!")