From 9e7a2656e86ff62cd92bd90ca2704bc5c0d609b1 Mon Sep 17 00:00:00 2001 From: Wei Luo Date: Mon, 9 Nov 2026 06:46:52 +0000 Subject: [PATCH 1/4] Parse catalog labels in REST load table and load view responses The REST spec adds an optional `labels` field to LoadTableResult and LoadViewResult for catalog-provided enrichment such as ownership or classification. PyIceberg dropped the field. Add `Labels` and `FieldLabel` models and parse `labels` on `TableResponse` and `ViewResponse`, matching the Java read path (apache/iceberg#18045). --- pyiceberg/catalog/rest/__init__.py | 3 ++ pyiceberg/table/labels.py | 53 +++++++++++++++++++++++ tests/catalog/test_rest.py | 18 ++++++++ tests/table/test_labels.py | 67 ++++++++++++++++++++++++++++++ 4 files changed, 141 insertions(+) create mode 100644 pyiceberg/table/labels.py create mode 100644 tests/table/test_labels.py diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index c8e2399fe8..95db70c2a7 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -88,6 +88,7 @@ TableIdentifier, TableProperties, ) +from pyiceberg.table.labels import Labels from pyiceberg.table.metadata import TableMetadata from pyiceberg.table.sorting import UNSORTED_SORT_ORDER, SortOrder, assign_fresh_sort_order_ids from pyiceberg.table.update import ( @@ -357,6 +358,7 @@ class TableResponse(IcebergBaseModel): metadata: TableMetadata config: Properties = Field(default_factory=dict) storage_credentials: list[StorageCredential] = Field(alias="storage-credentials", default_factory=list) + labels: Labels = Field(default_factory=Labels) class LoadCredentialsResponse(IcebergBaseModel): @@ -367,6 +369,7 @@ class ViewResponse(IcebergBaseModel): metadata_location: str = Field(..., alias="metadata-location") metadata: ViewMetadata config: Properties = Field(default_factory=dict) + labels: Labels = Field(default_factory=Labels) class CreateTableRequest(IcebergBaseModel): diff --git a/pyiceberg/table/labels.py b/pyiceberg/table/labels.py new file mode 100644 index 0000000000..d7663d9b78 --- /dev/null +++ b/pyiceberg/table/labels.py @@ -0,0 +1,53 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +from typing import Any + +from pydantic import Field, model_validator + +from pyiceberg.typedef import IcebergBaseModel + + +class FieldLabel(IcebergBaseModel): + """Labels attached to a single field, identified by its field ID.""" + + field_id: int = Field(alias="field-id") + labels: dict[str, str] + + +class Labels(IcebergBaseModel): + """Optional labels that a catalog returns with a table or view. + + Labels are enrichment supplied by the catalog, such as ownership or classification. + They are not table state and are not written to the table metadata. + """ + + object_labels: dict[str, str] = Field(alias="object-labels", default_factory=dict) + fields: list[FieldLabel] = Field(default_factory=list) + + @model_validator(mode="before") + @classmethod + def treat_null_as_absent(cls, data: Any) -> Any: + # A null value parses the same as a missing key. + if data is None: + return {} + if isinstance(data, dict): + return {key: value for key, value in data.items() if value is not None} + return data + + def is_empty(self) -> bool: + """Return True when there are neither object-level nor field-level labels.""" + return not self.object_labels and not self.fields diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index ad1e3c3296..535406f59f 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -51,6 +51,8 @@ HttpMethod, RestCatalog, ScanPlanningMode, + TableResponse, + ViewResponse, _RetryTimeoutHTTPAdapter, ) from pyiceberg.exceptions import ( @@ -71,6 +73,7 @@ from pyiceberg.partitioning import PartitionField, PartitionSpec from pyiceberg.schema import Schema from pyiceberg.table import Table +from pyiceberg.table.labels import FieldLabel, Labels from pyiceberg.table.metadata import TableMetadataV1 from pyiceberg.table.sorting import SortField, SortOrder from pyiceberg.transforms import IdentityTransform, TruncateTransform @@ -3713,3 +3716,18 @@ def test_load_table_without_storage_credentials( ) assert actual.metadata.model_dump() == expected.metadata.model_dump() assert actual == expected + + +def test_load_responses_parse_labels( + example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], example_view_metadata_rest_json: dict[str, Any] +) -> None: + labels_json = {"object-labels": {"owner": "team-a"}, "fields": [{"field-id": 1, "labels": {"classification": "pii"}}]} + expected = Labels(object_labels={"owner": "team-a"}, fields=[FieldLabel(field_id=1, labels={"classification": "pii"})]) + cases: list[tuple[type[TableResponse | ViewResponse], dict[str, Any]]] = [ + (TableResponse, example_table_metadata_with_snapshot_v1_rest_json), + (ViewResponse, example_view_metadata_rest_json), + ] + for response_type, payload in cases: + assert response_type.model_validate({**payload, "labels": labels_json}).labels == expected + assert response_type.model_validate({**payload, "labels": None}).labels == Labels() + assert response_type.model_validate(payload).labels == Labels() diff --git a/tests/table/test_labels.py b/tests/table/test_labels.py new file mode 100644 index 0000000000..49fab32712 --- /dev/null +++ b/tests/table/test_labels.py @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +import pytest +from pydantic import ValidationError + +from pyiceberg.table.labels import FieldLabel, Labels + + +def test_labels_from_json() -> None: + cases = [ + ('{"object-labels": null, "fields": null}', Labels()), + ( + '{"object-labels": {"owner": "team-a", "sensitivity": "high"}}', + Labels(object_labels={"owner": "team-a", "sensitivity": "high"}), + ), + ( + '{"fields": [{"field-id": 3, "labels": {"pii": "true"}}]}', + Labels(fields=[FieldLabel(field_id=3, labels={"pii": "true"})]), + ), + ( + """ + { + "object-labels": {"owner": "team-a"}, + "fields": [ + {"field-id": 1, "labels": {"classification": "pii"}}, + {"field-id": 2, "labels": {}} + ] + } + """, + Labels( + object_labels={"owner": "team-a"}, + fields=[FieldLabel(field_id=1, labels={"classification": "pii"}), FieldLabel(field_id=2, labels={})], + ), + ), + ] + for json, expected in cases: + assert Labels.model_validate_json(json) == expected + + +def test_labels_is_empty() -> None: + assert Labels().is_empty() + assert not Labels(object_labels={"owner": "team-a"}).is_empty() + assert not Labels(fields=[FieldLabel(field_id=1, labels={})]).is_empty() + + +def test_field_label_requires_field_id_and_labels() -> None: + cases = [ + ('{"labels": {}}', "1 validation error for FieldLabel\nfield-id\n Field required"), + ('{"field-id": 1}', "1 validation error for FieldLabel\nlabels\n Field required"), + ] + for json, message in cases: + with pytest.raises(ValidationError, match=message): + FieldLabel.model_validate_json(json) From d1145be9413c506ebd86157627b9a70b2e89bac3 Mon Sep 17 00:00:00 2001 From: Wei Luo Date: Tue, 10 Nov 2026 05:23:47 +0000 Subject: [PATCH 2/4] Read non-object labels as empty A string, list, number, or bool `labels` value failed the whole load. Java `LabelsParser` reads keys only under `hasNonNull`, so it returns empty labels for any non-object value. Do the same. --- pyiceberg/table/labels.py | 9 +++++---- tests/catalog/test_rest.py | 5 ++++- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/pyiceberg/table/labels.py b/pyiceberg/table/labels.py index d7663d9b78..104a3e25e6 100644 --- a/pyiceberg/table/labels.py +++ b/pyiceberg/table/labels.py @@ -41,12 +41,13 @@ class Labels(IcebergBaseModel): @model_validator(mode="before") @classmethod def treat_null_as_absent(cls, data: Any) -> Any: - # A null value parses the same as a missing key. - if data is None: - return {} + # A key with a null value parses the same as a missing key. if isinstance(data, dict): return {key: value for key, value in data.items() if value is not None} - return data + if isinstance(data, cls): + return data + # Like Java, a null or non-object value reads as empty labels. + return {} def is_empty(self) -> bool: """Return True when there are neither object-level nor field-level labels.""" diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 535406f59f..632dc7b5fb 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -3727,7 +3727,10 @@ def test_load_responses_parse_labels( (TableResponse, example_table_metadata_with_snapshot_v1_rest_json), (ViewResponse, example_view_metadata_rest_json), ] + # Like Java, a null or non-object value reads as empty labels. + invalid_values: list[Any] = [None, "oops", [], 1, True] for response_type, payload in cases: assert response_type.model_validate({**payload, "labels": labels_json}).labels == expected - assert response_type.model_validate({**payload, "labels": None}).labels == Labels() assert response_type.model_validate(payload).labels == Labels() + for invalid in invalid_values: + assert response_type.model_validate({**payload, "labels": invalid}).labels == Labels() From c93fa5ff9c45107bda0a3db1e74fd2cf03dbce62 Mon Sep 17 00:00:00 2001 From: Wei Luo Date: Mon, 9 Nov 2026 06:46:53 +0000 Subject: [PATCH 3/4] Expose catalog labels on Table Labels parsed from the load response were not reachable from the loaded table. Add `Table.labels`. The REST catalog sets it on load, create, and register. Other catalogs and staged tables (`create_table_transaction`) get empty labels. Labels are not table state: they are not part of equality, and refresh and commits keep them. This matches the Java `SupportsLabels` behavior (apache/iceberg#18046). --- pyiceberg/catalog/rest/__init__.py | 1 + pyiceberg/table/__init__.py | 5 +++++ tests/catalog/test_rest.py | 34 ++++++++++++++++++++++++++++++ tests/table/test_init.py | 5 +++++ 4 files changed, 45 insertions(+) diff --git a/pyiceberg/catalog/rest/__init__.py b/pyiceberg/catalog/rest/__init__.py index 95db70c2a7..ddd2f0577b 100644 --- a/pyiceberg/catalog/rest/__init__.py +++ b/pyiceberg/catalog/rest/__init__.py @@ -1169,6 +1169,7 @@ def _response_to_table(self, identifier_tuple: tuple[str, ...], table_response: ), catalog=self, config=table_response.config, + labels=table_response.labels, ) def _response_to_staged_table(self, identifier_tuple: tuple[str, ...], table_response: TableResponse) -> StagedTable: diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index df7aaf824f..bc4cff1d5a 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -59,6 +59,7 @@ from pyiceberg.schema import Schema from pyiceberg.table.delete_file_index import DeleteFileIndex from pyiceberg.table.inspect import InspectTable +from pyiceberg.table.labels import Labels from pyiceberg.table.locations import LocationProvider, load_location_provider from pyiceberg.table.maintenance import MaintenanceTable from pyiceberg.table.metadata import INITIAL_SEQUENCE_NUMBER, TableMetadata @@ -1345,6 +1346,8 @@ class Table: io: FileIO catalog: Catalog config: dict[str, str] + # Catalog-provided labels, set when the catalog loads the table. They are not table state, so refresh() and commits keep them. + labels: Labels def __init__( self, @@ -1355,6 +1358,7 @@ def __init__( catalog: Catalog, config: dict[str, str] = EMPTY_DICT, kms_client: KeyManagementClient | None = None, + labels: Labels | None = None, ) -> None: self._identifier = identifier self.metadata = metadata @@ -1363,6 +1367,7 @@ def __init__( self.catalog = catalog self.config = config self._kms_client = kms_client + self.labels = labels if labels is not None else Labels() self._encryption_manager: _CachedEncryptionManager | None = None @property diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index 632dc7b5fb..ca21ac8fd4 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -3734,3 +3734,37 @@ def test_load_responses_parse_labels( assert response_type.model_validate(payload).labels == Labels() for invalid in invalid_values: assert response_type.model_validate({**payload, "labels": invalid}).labels == Labels() + + +def test_load_table_with_labels(rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any]) -> None: + rest_mock.get( + f"{TEST_URI}v1/namespaces/fokko/tables/table", + json={ + **example_table_metadata_with_snapshot_v1_rest_json, + "labels": {"object-labels": {"owner": "team-a"}, "fields": [{"field-id": 1, "labels": {"classification": "pii"}}]}, + }, + status_code=200, + request_headers=TEST_HEADERS, + ) + catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + table = catalog.load_table(("fokko", "table")) + + expected = Labels(object_labels={"owner": "team-a"}, fields=[FieldLabel(field_id=1, labels={"classification": "pii"})]) + assert table.labels == expected + + # Labels are not table state, so equality ignores them and refresh() keeps the labels from the first load. + assert table == Table( + identifier=("fokko", "table"), + metadata_location=example_table_metadata_with_snapshot_v1_rest_json["metadata-location"], + metadata=TableMetadataV1(**example_table_metadata_with_snapshot_v1_rest_json["metadata"]), + io=load_file_io(), + catalog=catalog, + ) + rest_mock.get( + f"{TEST_URI}v1/namespaces/fokko/tables/table", + json={**example_table_metadata_with_snapshot_v1_rest_json, "labels": {"object-labels": {"owner": "team-b"}}}, + status_code=200, + request_headers=TEST_HEADERS, + ) + table.refresh() + assert table.labels == expected diff --git a/tests/table/test_init.py b/tests/table/test_init.py index db6dc00e39..c6b87e7335 100644 --- a/tests/table/test_init.py +++ b/tests/table/test_init.py @@ -46,6 +46,7 @@ Table, TableIdentifier, ) +from pyiceberg.table.labels import Labels from pyiceberg.table.metadata import EncryptedKey, TableMetadataUtil, TableMetadataV1, TableMetadataV2, _generate_snapshot_id from pyiceberg.table.refs import MAIN_BRANCH, SnapshotRef, SnapshotRefType from pyiceberg.table.snapshots import ( @@ -193,6 +194,10 @@ def test_location(table_v2: Table) -> None: assert table_v2.location() == "s3://bucket/test/location" +def test_labels_default_to_empty(table_v2: Table) -> None: + assert table_v2.labels == Labels() + + def test_current_snapshot(table_v2: Table) -> None: assert table_v2.current_snapshot() == Snapshot( snapshot_id=3055729675574597004, From f1d11c0836f774c4bf1cf260b0fa820b1129d176 Mon Sep 17 00:00:00 2001 From: Wei Luo Date: Tue, 10 Nov 2026 05:25:26 +0000 Subject: [PATCH 4/4] Test labels on create, register, and pickle Create and register hand labels to the same `_response_to_table` as load, but only load was tested. Check all three. Also pin that a pickled table keeps its labels, the one intended difference from Java, where `BaseTable.labels` is transient. --- tests/catalog/test_rest.py | 31 +++++++++++++++++++------------ 1 file changed, 19 insertions(+), 12 deletions(-) diff --git a/tests/catalog/test_rest.py b/tests/catalog/test_rest.py index ca21ac8fd4..37508e9a64 100644 --- a/tests/catalog/test_rest.py +++ b/tests/catalog/test_rest.py @@ -19,6 +19,7 @@ import base64 import os +import pickle from collections.abc import Callable, Iterator from contextlib import contextmanager from typing import Any, cast @@ -3736,21 +3737,27 @@ def test_load_responses_parse_labels( assert response_type.model_validate({**payload, "labels": invalid}).labels == Labels() -def test_load_table_with_labels(rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any]) -> None: - rest_mock.get( - f"{TEST_URI}v1/namespaces/fokko/tables/table", - json={ - **example_table_metadata_with_snapshot_v1_rest_json, - "labels": {"object-labels": {"owner": "team-a"}, "fields": [{"field-id": 1, "labels": {"classification": "pii"}}]}, - }, - status_code=200, - request_headers=TEST_HEADERS, - ) +def test_table_labels_from_rest_responses( + rest_mock: Mocker, example_table_metadata_with_snapshot_v1_rest_json: dict[str, Any], table_schema_simple: Schema +) -> None: + response = { + **example_table_metadata_with_snapshot_v1_rest_json, + "labels": {"object-labels": {"owner": "team-a"}, "fields": [{"field-id": 1, "labels": {"classification": "pii"}}]}, + } + rest_mock.get(f"{TEST_URI}v1/namespaces/fokko/tables/table", json=response, status_code=200, request_headers=TEST_HEADERS) + rest_mock.post(f"{TEST_URI}v1/namespaces/fokko/tables", json=response, status_code=200, request_headers=TEST_HEADERS) + rest_mock.post(f"{TEST_URI}v1/namespaces/fokko/register", json=response, status_code=200, request_headers=TEST_HEADERS) catalog = RestCatalog("rest", uri=TEST_URI, token=TEST_TOKEN) + expected = Labels(object_labels={"owner": "team-a"}, fields=[FieldLabel(field_id=1, labels={"classification": "pii"})]) + table = catalog.load_table(("fokko", "table")) + created = catalog.create_table(("fokko", "table"), schema=table_schema_simple) + registered = catalog.register_table(("fokko", "table"), metadata_location="s3://warehouse/database/table/metadata.json") + for actual in (table, created, registered): + assert actual.labels == expected - expected = Labels(object_labels={"owner": "team-a"}, fields=[FieldLabel(field_id=1, labels={"classification": "pii"})]) - assert table.labels == expected + # A pickled table keeps its labels. Java does not serialize them, because BaseTable.labels is transient. + assert pickle.loads(pickle.dumps(table)).labels == expected # Labels are not table state, so equality ignores them and refresh() keeps the labels from the first load. assert table == Table(