From 71e4d84b2cc3788229dbaaeddd01902c2e96cd93 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Thu, 14 Nov 2024 03:12:13 +0000 Subject: [PATCH 1/4] make metadata_location optional --- pyiceberg/catalog/rest.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pyiceberg/catalog/rest.py b/pyiceberg/catalog/rest.py index bcfa46b7a7..2b48330bfc 100644 --- a/pyiceberg/catalog/rest.py +++ b/pyiceberg/catalog/rest.py @@ -156,7 +156,7 @@ def _retry_hook(retry_state: RetryCallState) -> None: class TableResponse(IcebergBaseModel): - metadata_location: Optional[str] = Field(alias="metadata-location") + metadata_location: Optional[str] = Field(alias="metadata-location", default=None) metadata: TableMetadata config: Properties = Field(default_factory=dict) @@ -599,7 +599,6 @@ def _create_table( response.raise_for_status() except HTTPError as exc: self._handle_non_200_response(exc, {409: TableAlreadyExistsError}) - return TableResponse(**response.json()) @retry(**_RETRY_ARGS) From 3e0445579f944bc678992c2ba8d8ea2274b6222a Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Thu, 14 Nov 2024 18:31:53 +0000 Subject: [PATCH 2/4] add test for staged creation --- dev/docker-compose-integration.yml | 2 +- iceberg | 1 + tests/catalog/test_rest.py | 75 ++++++++++++++++++++++++++++++ 3 files changed, 77 insertions(+), 1 deletion(-) create mode 160000 iceberg diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index fccdcdc757..9a807fca3f 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -41,7 +41,7 @@ services: - hive:hive - minio:minio rest: - image: tabulario/iceberg-rest + image: apache/iceberg-rest-adapter container_name: pyiceberg-rest networks: iceberg_net: diff --git a/iceberg b/iceberg new file mode 160000 index 0000000000..449f616e26 --- /dev/null +++ b/iceberg @@ -0,0 +1 @@ +Subproject commit 449f616e26f423dc73f98f2c545a5dc79ca1e190 diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 9d75154ae0..f8662c1bf4 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -79,6 +79,17 @@ def example_table_metadata_with_snapshot_v1_rest_json(example_table_metadata_wit } +@pytest.fixture +def example_table_metadata_with_no_location(example_table_metadata_with_snapshot_v1: Dict[str, Any]) -> Dict[str, Any]: + return { + "metadata": example_table_metadata_with_snapshot_v1, + "config": { + "client.factory": "io.tabular.iceberg.catalog.TabularAwsClientFactory", + "region": "us-west-2", + }, + } + + @pytest.fixture def example_table_metadata_no_snapshot_v1_rest_json(example_table_metadata_no_snapshot_v1: Dict[str, Any]) -> Dict[str, Any]: return { @@ -899,6 +910,70 @@ def test_create_table_with_given_location_removes_trailing_slash_200( assert rest_mock.last_request.json()["location"] == location +def test_create_staged_table_200( + rest_mock: Mocker, + table_schema_simple: Schema, + example_table_metadata_with_no_location: Dict[str, Any], + example_table_metadata_no_snapshot_v1_rest_json: Dict[str, Any], +) -> None: + rest_mock.post( + f"{TEST_URI}v1/namespaces/fokko/tables", + json=example_table_metadata_with_no_location, + status_code=200, + request_headers=TEST_HEADERS, + ) + rest_mock.post( + f"{TEST_URI}v1/namespaces/fokko/tables/fokko2", + json=example_table_metadata_no_snapshot_v1_rest_json, + status_code=200, + request_headers=TEST_HEADERS, + ) + identifier = ("fokko", "fokko2") + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + txn = catalog.create_table_transaction( + identifier=identifier, + schema=table_schema_simple, + location=None, + partition_spec=PartitionSpec( + PartitionField(source_id=1, field_id=1000, transform=TruncateTransform(width=3), name="id"), spec_id=1 + ), + sort_order=SortOrder(SortField(source_id=2, transform=IdentityTransform())), + properties={"owner": "fokko"}, + ) + txn.commit_transaction() + + actual_response = rest_mock.last_request.json() + expected = { + "identifier": {"namespace": ["fokko"], "name": "fokko2"}, + "requirements": [{"type": "assert-create"}], + "updates": [ + {"action": "assign-uuid", "uuid": "b55d9dda-6561-423a-8bfc-787980ce421f"}, + {"action": "upgrade-format-version", "format-version": 1}, + { + "action": "add-schema", + "schema": { + "type": "struct", + "fields": [ + {"id": 1, "name": "id", "type": "int", "required": False}, + {"id": 2, "name": "data", "type": "string", "required": False}, + ], + "schema-id": 0, + "identifier-field-ids": [], + }, + "last-column-id": 2, + }, + {"action": "set-current-schema", "schema-id": -1}, + {"action": "add-spec", "spec": {"spec-id": 0, "fields": []}}, + {"action": "set-default-spec", "spec-id": -1}, + {"action": "add-sort-order", "sort-order": {"order-id": 0, "fields": []}}, + {"action": "set-default-sort-order", "sort-order-id": -1}, + {"action": "set-location", "location": "s3://warehouse/database/table"}, + {"action": "set-properties", "updates": {"owner": "bryan", "write.metadata.compression-codec": "gzip"}}, + ], + } + assert actual_response == expected + + def test_create_table_409(rest_mock: Mocker, table_schema_simple: Schema) -> None: rest_mock.post( f"{TEST_URI}v1/namespaces/fokko/tables", From 2b864d04fd75a02aaefa064f3c4af5d1cfc14869 Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Thu, 14 Nov 2024 18:32:45 +0000 Subject: [PATCH 3/4] revert --- dev/docker-compose-integration.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/dev/docker-compose-integration.yml b/dev/docker-compose-integration.yml index 9a807fca3f..fccdcdc757 100644 --- a/dev/docker-compose-integration.yml +++ b/dev/docker-compose-integration.yml @@ -41,7 +41,7 @@ services: - hive:hive - minio:minio rest: - image: apache/iceberg-rest-adapter + image: tabulario/iceberg-rest container_name: pyiceberg-rest networks: iceberg_net: From ac3215e472afd29951ac94675fe0fbbfbcbdf83d Mon Sep 17 00:00:00 2001 From: Sung Yun <107272191+sungwy@users.noreply.github.com> Date: Thu, 14 Nov 2024 19:54:45 +0000 Subject: [PATCH 4/4] revert from testing --- iceberg | 1 - 1 file changed, 1 deletion(-) delete mode 160000 iceberg diff --git a/iceberg b/iceberg deleted file mode 160000 index 449f616e26..0000000000 --- a/iceberg +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 449f616e26f423dc73f98f2c545a5dc79ca1e190