FFI and Julia Bindings for incremental scan changes

.github/workflows/test.yml

@@ -61,10 +61,13 @@ jobs:
               -e MINIO_ROOT_USER=root \
               -e MINIO_ROOT_PASSWORD=password \
               -e MINIO_DOMAIN=minio \
-              -v ${{ github.workspace }}/assets/tpch:/input minio/minio:latest server /data/
+              -v ${{ github.workspace }}/assets/tpch:/input \
+              -v ${{ github.workspace }}/assets/incremental:/input_incremental \
+              minio/minio:latest server /data/
             until (docker exec minio mc alias set minio http://localhost:9000 root password) do echo '... waiting ...' && sleep 1; done;
             docker exec minio mc mb minio/warehouse
             docker exec minio mc cp -r /input/tpch.sf01/ minio/warehouse/tpch.sf01/
+            docker exec minio mc cp -r /input_incremental/ minio/warehouse/incremental/;
             docker run -d --name rest \
               --network=iceberg_net \
               -p 8181:8181 \

.gitignore

@@ -1,5 +1,7 @@
 **/.env
 iceberg_rust_ffi/target
+iceberg_rust_ffi/target/**/*
 iceberg_rust_ffi/integration_test
 **/*.dylib
-**/.claude
+**/.claude
+**/.DS_Store

CLAUDE.md

@@ -34,11 +34,6 @@ This project provides a **Foreign Function Interface (FFI)** for Apache Iceberg,
 - **Julia Integration**: Conditional compilation features for Julia interop (`julia` feature flag)
 - **Memory Management**: Safe FFI patterns with proper cleanup functions
 
-#### C header (`include/iceberg_rust_ffi.h`)
-- **Manual Generation**: C header is not generated right now. Whenever you make a change in the Rust library, examine whether the header should be updated.
-- **C99 Compatible**: Ensures compatibility with standard C compilers
-- **Response Structures**: Async operations return response structures with context for cancellation
-
 ### FFI Design Patterns
 
 #### Async Operations with Callbacks

Makefile

@@ -48,7 +48,7 @@ test: build
 		exit 1; \
 	fi
 	@set -a && . ./.env && set +a && \
-		export ICEBERG_RUST_LIB=$(TARGET_DIR) && \
+		export ICEBERG_RUST_LIB=$$(pwd)/$(TARGET_DIR) && \
 		$(JULIA_THREADS_ENV) julia --project=. -e 'using Pkg; Pkg.test()'
 
 # Start Julia REPL with environment configured (requires .env file)
@@ -58,7 +58,7 @@ repl: build
 		exit 1; \
 	fi
 	@set -a && . ./.env && set +a && \
-		export ICEBERG_RUST_LIB=$(TARGET_DIR) && \
+		export ICEBERG_RUST_LIB=$$(pwd)/$(TARGET_DIR) && \
 		$(JULIA_THREADS_ENV) julia --project=.
 
 # Clean build artifacts

assets/incremental/test1/metadata/00000-9b1f6e90-9146-4446-bc93-9625ad2f12de.metadata.json

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"bc0f34cc-7492-4fb0-bd26-7e6749d1aed3","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761916004464,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:06:44.221713590Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":2201372158267155259,"refs":{"main":{"snapshot-id":2201372158267155259,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":2201372158267155259,"timestamp-ms":1761916004464,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-2201372158267155259-1-565b26e5-f200-448f-8ccf-02cebeb64f15.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916004464,"snapshot-id":2201372158267155259}],"metadata-log":[]}

assets/incremental/test1/metadata/00000-cf659803-d5fa-44fe-9c86-ecfc225f11ce.metadata.json

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"8d69da49-0398-4107-8ede-956e3c1031ed","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761915547394,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T12:59:06.941439875Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":7624749623479285340,"refs":{"main":{"snapshot-id":7624749623479285340,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":7624749623479285340,"timestamp-ms":1761915547394,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-7624749623479285340-1-06af070a-6938-43ea-94f6-659a8b7b6799.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761915547394,"snapshot-id":7624749623479285340}],"metadata-log":[]}

assets/incremental/test1/metadata/00000-d4d80e79-dc1a-4a50-a636-25cf30946d09.metadata.json

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"208df1e5-aa98-49e4-91f8-0fff75d991ed","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761915998275,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:06:38.150855545Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":935682165401343509,"refs":{"main":{"snapshot-id":935682165401343509,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":935682165401343509,"timestamp-ms":1761915998275,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-935682165401343509-1-94811a1f-2293-4ebd-a227-cf4fc471fe85.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761915998275,"snapshot-id":935682165401343509}],"metadata-log":[]}

assets/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":1,"last-updated-ms":1761916064879,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":6544470421042678943,"refs":{"main":{"snapshot-id":6544470421042678943,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943}],"metadata-log":[]}

assets/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":2,"last-updated-ms":1761916192089,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":6540713100348352610,"refs":{"main":{"snapshot-id":6540713100348352610,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0},{"sequence-number":2,"snapshot-id":6540713100348352610,"parent-snapshot-id":6544470421042678943,"timestamp-ms":1761916192089,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2433","changed-partition-count":"1","total-records":"109","total-files-size":"4637","total-data-files":"10","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943},{"timestamp-ms":1761916192089,"snapshot-id":6540713100348352610}],"metadata-log":[{"timestamp-ms":1761916064879,"metadata-file":"s3://warehouse/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json"}]}

assets/incremental/test1/metadata/00002-ec6c7e37-0b8f-44b3-bb11-7fee9a9091ca.metadata.json

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":3,"last-updated-ms":1761916215684,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":8895755745308925611,"refs":{"main":{"snapshot-id":8895755745308925611,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0},{"sequence-number":2,"snapshot-id":6540713100348352610,"parent-snapshot-id":6544470421042678943,"timestamp-ms":1761916192089,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2433","changed-partition-count":"1","total-records":"109","total-files-size":"4637","total-data-files":"10","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro","schema-id":0},{"sequence-number":3,"snapshot-id":8895755745308925611,"parent-snapshot-id":6540713100348352610,"timestamp-ms":1761916215684,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2490","changed-partition-count":"1","total-records":"208","total-files-size":"7127","total-data-files":"15","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-8895755745308925611-1-2e224b4d-1419-4378-b5ae-76f3ee5e039d.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943},{"timestamp-ms":1761916192089,"snapshot-id":6540713100348352610},{"timestamp-ms":1761916215684,"snapshot-id":8895755745308925611}],"metadata-log":[{"timestamp-ms":1761916064879,"metadata-file":"s3://warehouse/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json"},{"timestamp-ms":1761916192089,"metadata-file":"s3://warehouse/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json"}]}

assets/incremental/test1/metadata/00003-359e8bb8-1e5d-46d2-bcde-fdaeaa41114f.metadata.json

@@ -0,0 +1 @@
+{"format-version":2,"table-uuid":"2b9105aa-1518-4c17-975d-0ba0a8232f43","location":"s3://warehouse/incremental/test1","last-sequence-number":4,"last-updated-ms":1761916266468,"last-column-id":1,"current-schema-id":0,"schemas":[{"type":"struct","schema-id":0,"fields":[{"id":1,"name":"n","required":false,"type":"long"}]}],"default-spec-id":0,"partition-specs":[{"spec-id":0,"fields":[]}],"last-partition-id":999,"default-sort-order-id":0,"sort-orders":[{"order-id":0,"fields":[]}],"properties":{"owner":"root","created-at":"2025-10-31T13:07:44.764295424Z","write.delete.mode":"merge-on-read","write.parquet.compression-codec":"zstd"},"current-snapshot-id":6832180054960511692,"refs":{"main":{"snapshot-id":6832180054960511692,"type":"branch"}},"snapshots":[{"sequence-number":1,"snapshot-id":6544470421042678943,"timestamp-ms":1761916064879,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"10","added-files-size":"2204","changed-partition-count":"1","total-records":"10","total-files-size":"2204","total-data-files":"5","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6544470421042678943-1-154380bf-11a5-4ac4-8c8f-6ade7758b93c.avro","schema-id":0},{"sequence-number":2,"snapshot-id":6540713100348352610,"parent-snapshot-id":6544470421042678943,"timestamp-ms":1761916192089,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2433","changed-partition-count":"1","total-records":"109","total-files-size":"4637","total-data-files":"10","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6540713100348352610-1-13d31e71-0067-4bf5-92a2-2bb1b51cc181.avro","schema-id":0},{"sequence-number":3,"snapshot-id":8895755745308925611,"parent-snapshot-id":6540713100348352610,"timestamp-ms":1761916215684,"summary":{"operation":"append","spark.app.id":"local-1761914226119","added-data-files":"5","added-records":"99","added-files-size":"2490","changed-partition-count":"1","total-records":"208","total-files-size":"7127","total-data-files":"15","total-delete-files":"0","total-position-deletes":"0","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-8895755745308925611-1-2e224b4d-1419-4378-b5ae-76f3ee5e039d.avro","schema-id":0},{"sequence-number":4,"snapshot-id":6832180054960511692,"parent-snapshot-id":8895755745308925611,"timestamp-ms":1761916266468,"summary":{"operation":"delete","spark.app.id":"local-1761914226119","added-position-delete-files":"2","added-delete-files":"2","added-files-size":"2924","added-position-deletes":"2","changed-partition-count":"1","total-records":"208","total-files-size":"10051","total-data-files":"15","total-delete-files":"2","total-position-deletes":"2","total-equality-deletes":"0","engine-version":"3.5.5","app-id":"local-1761914226119","engine-name":"spark","iceberg-version":"Apache Iceberg 1.8.1 (commit 9ce0fcf0af7becf25ad9fc996c3bad2afdcfd33d)"},"manifest-list":"s3://warehouse/incremental/test1/metadata/snap-6832180054960511692-1-f6b39896-f4f2-43d8-8f02-990884cd7287.avro","schema-id":0}],"statistics":[],"partition-statistics":[],"snapshot-log":[{"timestamp-ms":1761916064879,"snapshot-id":6544470421042678943},{"timestamp-ms":1761916192089,"snapshot-id":6540713100348352610},{"timestamp-ms":1761916215684,"snapshot-id":8895755745308925611},{"timestamp-ms":1761916266468,"snapshot-id":6832180054960511692}],"metadata-log":[{"timestamp-ms":1761916064879,"metadata-file":"s3://warehouse/incremental/test1/metadata/00000-fa2b5723-9c14-4ea2-bb9f-d39644a8849f.metadata.json"},{"timestamp-ms":1761916192089,"metadata-file":"s3://warehouse/incremental/test1/metadata/00001-43d0a4ca-f14a-44ec-ba6f-9dcf4ab2b8db.metadata.json"},{"timestamp-ms":1761916215684,"metadata-file":"s3://warehouse/incremental/test1/metadata/00002-ec6c7e37-0b8f-44b3-bb11-7fee9a9091ca.metadata.json"}]}