From 4d2ac7a3f3573d19729ca33d3c3ea055ae76988e Mon Sep 17 00:00:00 2001 From: Xiening Dai Date: Sat, 14 Nov 2026 18:18:13 +0000 Subject: [PATCH] feat(util): add location resolve and relativize helpers Add LocationUtil::HasScheme, ResolveLocation and RelativizeLocation for v4 relative paths, as defined in the spec's "Paths in Metadata" section. - HasScheme is strict RFC 3986 scheme detection on every platform. - ResolveLocation returns absolute paths unchanged and joins relative paths to the table location with "/", with no normalization or URI parsing. If the table location already ends in "/", no second separator is added, since object stores such as S3 treat "//" as part of the key. - RelativizeLocation strips the table location and one separator when a path is under it, and otherwise returns the path unchanged. Tests cover every example in the spec's path resolution table, plus trailing slashes, sibling prefixes, schemes, escaped characters and round trips. Closes #1123 --- src/iceberg/test/location_util_test.cc | 157 +++++++++++++++++++++++++ src/iceberg/util/location_util.cc | 59 ++++++++++ src/iceberg/util/location_util.h | 27 +++++ 3 files changed, 243 insertions(+) diff --git a/src/iceberg/test/location_util_test.cc b/src/iceberg/test/location_util_test.cc index 0b806345c..7ef1a32a7 100644 --- a/src/iceberg/test/location_util_test.cc +++ b/src/iceberg/test/location_util_test.cc @@ -85,4 +85,161 @@ TEST(LocationUtilTest, ParseScheme) { #endif } +TEST(LocationUtilTest, HasScheme) { + EXPECT_TRUE(LocationUtil::HasScheme("s3://bucket/db/table")); + EXPECT_TRUE(LocationUtil::HasScheme("hdfs://wh/db/table")); + EXPECT_TRUE(LocationUtil::HasScheme("file:/tmp/table")); + EXPECT_TRUE(LocationUtil::HasScheme("S3A://bucket/path")); + EXPECT_TRUE(LocationUtil::HasScheme("a+b-c.d1:rest")); + // Strict on every platform, unlike ParseScheme: a drive letter is a scheme. + EXPECT_TRUE(LocationUtil::HasScheme("C:/tmp/file.parquet")); + EXPECT_TRUE(LocationUtil::HasScheme("x:")); + + EXPECT_FALSE(LocationUtil::HasScheme("")); + EXPECT_FALSE(LocationUtil::HasScheme("data/00000-0.parquet")); + EXPECT_FALSE(LocationUtil::HasScheme("/wh/db/table")); + EXPECT_FALSE(LocationUtil::HasScheme("://bucket/path")); + EXPECT_FALSE(LocationUtil::HasScheme(":")); + // The first character must be a letter. + EXPECT_FALSE(LocationUtil::HasScheme("1s3://bucket")); + EXPECT_FALSE(LocationUtil::HasScheme("+s3://bucket")); + // Characters outside the scheme grammar before the colon. + EXPECT_FALSE(LocationUtil::HasScheme("data/part:1.parquet")); + EXPECT_FALSE(LocationUtil::HasScheme("my_scheme://bucket")); + EXPECT_FALSE(LocationUtil::HasScheme("\\\\?\\C:\\long\\file.parquet")); +} + +// Every example from the spec's "Path Resolution" table. +TEST(LocationUtilTest, ResolveLocationSpecExamples) { + // Relative path: parts are joined on `/`. + EXPECT_EQ(LocationUtil::ResolveLocation("s3://bucket/db/table", "data/00000-0.parquet"), + "s3://bucket/db/table/data/00000-0.parquet"); + // Absolute path is used. + EXPECT_EQ(LocationUtil::ResolveLocation("s3://bucket/db/table", + "hdfs://wh/db/table/data/00000-0.parquet"), + "hdfs://wh/db/table/data/00000-0.parquet"); + // Duplicate separator: the table location ends in `/`, which the spec + // recommends against. No second separator is added (see ResolveLocation). + EXPECT_EQ( + LocationUtil::ResolveLocation("s3://bucket/db/table/", "data/00000-0.parquet"), + "s3://bucket/db/table/data/00000-0.parquet"); + // Duplicate separator: the relative path starts with `/`. + EXPECT_EQ( + LocationUtil::ResolveLocation("s3://bucket/db/table", "/data/00000-0.parquet"), + "s3://bucket/db/table//data/00000-0.parquet"); + // Fully-qualified (v3 and earlier) path is used. + EXPECT_EQ(LocationUtil::ResolveLocation("s3://bucket/db/table", + "s3://bucket/db/table/data/00000-0.parquet"), + "s3://bucket/db/table/data/00000-0.parquet"); + // Missing scheme (v3 and earlier): prepending a scheme to a pre-v4 path is + // not resolution and is not done here; such a path is not absolute. + EXPECT_FALSE(LocationUtil::HasScheme("/wh/db/table/data/00000-0.parquet")); +} + +TEST(LocationUtilTest, ResolveLocation) { + // No `.` or `..` normalization. + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", "./data/f.parquet"), + "s3://b/t/./data/f.parquet"); + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", "../other/f.parquet"), + "s3://b/t/../other/f.parquet"); + // A trailing separator on the table location is not doubled; only one + // trailing separator is consumed, and a leading one on the path is kept. + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t/", "f.parquet"), + "s3://b/t/f.parquet"); + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t//", "f.parquet"), + "s3://b/t//f.parquet"); + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t/", "/f.parquet"), + "s3://b/t//f.parquet"); + // A scheme-less table location is joined the same way. + EXPECT_EQ(LocationUtil::ResolveLocation("/wh/t", "data/f.parquet"), + "/wh/t/data/f.parquet"); + // A different scheme is absolute. + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", "gs://other/f.parquet"), + "gs://other/f.parquet"); + // A colon after the first segment does not make a path absolute. + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", "data/a:b.parquet"), + "s3://b/t/data/a:b.parquet"); + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", ""), "s3://b/t/"); +} + +TEST(LocationUtilTest, RelativizeLocation) { + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://bucket/db/table", + "s3://bucket/db/table/data/00000-0.parquet"), + "data/00000-0.parquet"); + // Only one separator is removed. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://bucket/db/table", + "s3://bucket/db/table//data/f.parquet"), + "/data/f.parquet"); + // Outside the table location: unchanged. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://bucket/db/table", + "s3://bucket/other/f.parquet"), + "s3://bucket/other/f.parquet"); + // A sibling that shares the prefix is not under the table location. + EXPECT_EQ(LocationUtil::RelativizeLocation("/t", "/t2/data/f.parquet"), + "/t2/data/f.parquet"); + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t", "s3://b/t2/f.parquet"), + "s3://b/t2/f.parquet"); + // The table location itself is not under it. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t", "s3://b/t"), "s3://b/t"); + // A different scheme does not match. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t", "s3a://b/t/f.parquet"), + "s3a://b/t/f.parquet"); + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t", "gs://b/t/f.parquet"), + "gs://b/t/f.parquet"); + // Matching is case-sensitive. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t", "S3://b/t/f.parquet"), + "S3://b/t/f.parquet"); + // A table location ending in `/` supplies the separator itself. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t/", "s3://b/t/f.parquet"), + "f.parquet"); + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t/", "s3://b/t//f.parquet"), + "/f.parquet"); + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t/", "s3://b/t/"), "s3://b/t/"); + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t/", "s3://b/t2/f.parquet"), + "s3://b/t2/f.parquet"); + // Already relative: unchanged. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t", "data/f.parquet"), + "data/f.parquet"); +} + +// Paths are opaque strings: percent-encoded, reserved and non-ASCII +// characters are copied as-is, never decoded or re-encoded. +TEST(LocationUtilTest, EscapedCharactersArePreserved) { + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", "data/a%20b%2Fc.parquet"), + "s3://b/t/data/a%20b%2Fc.parquet"); + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/my%20t", "data/x=a b/f.parquet"), + "s3://b/my%20t/data/x=a b/f.parquet"); + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", "data/k=v?#&+/f.parquet"), + "s3://b/t/data/k=v?#&+/f.parquet"); + EXPECT_EQ(LocationUtil::ResolveLocation("s3://b/t", "data/\xc3\xa9t\xc3\xa9.parquet"), + "s3://b/t/data/\xc3\xa9t\xc3\xa9.parquet"); + + EXPECT_EQ( + LocationUtil::RelativizeLocation("s3://b/my%20t", "s3://b/my%20t/a%2Fb.parquet"), + "a%2Fb.parquet"); + // An encoded and a decoded form are different paths: no match. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/my%20t", "s3://b/my t/f.parquet"), + "s3://b/my t/f.parquet"); + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/my t", "s3://b/my%20t/f.parquet"), + "s3://b/my%20t/f.parquet"); + // An encoded separator is not a separator. + EXPECT_EQ(LocationUtil::RelativizeLocation("s3://b/t", "s3://b/t%2Ff.parquet"), + "s3://b/t%2Ff.parquet"); +} + +TEST(LocationUtilTest, RelativizeThenResolveRoundTrips) { + for (std::string_view table : {"s3://bucket/db/table", "s3://bucket/db/table/"}) { + for (std::string_view path : + {"s3://bucket/db/table/data/00000-0.parquet", + "s3://bucket/db/table/metadata/snap-1.parquet", + "s3://bucket/db/table//x.parquet", "s3://bucket/other/f.parquet", + "hdfs://wh/db/table/f.parquet"}) { + EXPECT_EQ(LocationUtil::ResolveLocation( + table, LocationUtil::RelativizeLocation(table, path)), + path) + << "table: " << table; + } + } +} + } // namespace iceberg diff --git a/src/iceberg/util/location_util.cc b/src/iceberg/util/location_util.cc index 2a442a89e..2b3bc5ef4 100644 --- a/src/iceberg/util/location_util.cc +++ b/src/iceberg/util/location_util.cc @@ -66,4 +66,63 @@ std::string_view LocationUtil::ParseScheme(std::string_view location) { return candidate; } +bool LocationUtil::HasScheme(std::string_view location) { + // RFC 3986 section 3.1: a URI starts with `scheme ":"`, so the scheme is + // everything before the first colon. The spec ("Paths in Metadata") calls a + // path absolute iff it starts with such a scheme, as Java's + // LocationUtil.hasScheme does. + const auto colon = location.find(':'); + return colon != std::string_view::npos && IsValidScheme(location.substr(0, colon)); +} + +std::string LocationUtil::ResolveLocation(std::string_view table_location, + std::string_view location) { + if (HasScheme(location)) { + return std::string(location); + } + // The spec ("Path Resolution", + // https://iceberg.apache.org/spec/#path-resolution) defines this as a plain + // string join with `/`. + // It is not RFC 3986 reference resolution (section 5.2), which the spec + // rules out: a URI library would remove `.`/`..` segments, merge `//`, and + // could decode or re-encode `%XX`, producing a path that differs from the + // one the writer stored. Both parts are opaque strings in whatever encoding + // the writer used, and they are copied byte for byte. + // + // The spec recommends a table location that does not end in `/`, because + // the join would otherwise produce `//`. This applies that recommendation: + // if the table location already ends in `/`, no second separator is added, + // since object stores such as S3 treat `//` as part of the key and advise + // against it. A leading `/` on the relative path is kept, since it comes + // from a stored path that really contains `//`. + std::string resolved; + resolved.reserve(table_location.size() + 1 + location.size()); + resolved.append(table_location); + if (!table_location.ends_with('/')) { + resolved.push_back('/'); + } + resolved.append(location); + return resolved; +} + +std::string_view LocationUtil::RelativizeLocation(std::string_view table_location, + std::string_view location) { + // The spec ("Path Relativization") defines this as a plain prefix match on + // the table location followed by `/`, so a byte comparison is exact. No URI + // parsing or normalization: paths that only differ in encoding or `//` are + // different paths, and are kept absolute. A table location that already + // ends in `/` supplies the separator itself, matching ResolveLocation. + if (table_location.ends_with('/')) { + if (location.size() > table_location.size() && location.starts_with(table_location)) { + return location.substr(table_location.size()); + } + return location; + } + if (location.size() > table_location.size() && location.starts_with(table_location) && + location[table_location.size()] == '/') { + return location.substr(table_location.size() + 1); + } + return location; +} + } // namespace iceberg diff --git a/src/iceberg/util/location_util.h b/src/iceberg/util/location_util.h index 2a6147eed..d1b91df55 100644 --- a/src/iceberg/util/location_util.h +++ b/src/iceberg/util/location_util.h @@ -22,6 +22,7 @@ /// \file iceberg/util/location_util.h /// \brief Provide location path utilities. +#include #include #include "iceberg/iceberg_export.h" @@ -39,6 +40,32 @@ class ICEBERG_EXPORT LocationUtil { /// not validated. static std::string_view ParseScheme(std::string_view location); + /// \brief Whether a location starts with a URI scheme. + /// + /// This is the spec's test for an absolute path (see "Paths in Metadata"): + /// the text before the first colon must be a non-empty RFC 3986 scheme. It + /// is strict on every platform: unlike ParseScheme, a single letter such as + /// `C:` counts as a scheme. + static bool HasScheme(std::string_view location); + + /// \brief Resolve a location against a table location. + /// + /// A location with a URI scheme is absolute and is returned unchanged. + /// Otherwise the result is `table_location + "/" + location`, with no `/` + /// added when the table location already ends in one. The join does not + /// interpret `.` or `..`, and a location starting with `/` yields `//`. + static std::string ResolveLocation(std::string_view table_location, + std::string_view location); + + /// \brief Relativize a location against a table location. + /// + /// If the location starts with the table location immediately followed by + /// `/`, the remainder after that separator is returned. Otherwise the + /// location is returned unchanged. A table location that ends in `/` + /// already includes the separator. + static std::string_view RelativizeLocation(std::string_view table_location, + std::string_view location); + static std::string_view StripTrailingSlash(std::string_view path) { if (path.empty()) { return "";