Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
c447c1e
fix: keep URL passwords and SAS queries out of Azure store errors and…
dwsmith1983 Oct 20, 2026
65b5ad2
fix: keep ABFS URL passwords out of store keys, resolution errors and…
dwsmith1983 Oct 20, 2026
5fbb8fd
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 21, 2026
a3275e6
fix: name only the scheme when an Azure store gets a non-Azure URL
dwsmith1983 Oct 21, 2026
4c8da96
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 21, 2026
8acc4a2
docs: keep only the comments that explain the URL redaction rules
dwsmith1983 Oct 22, 2026
a1b30cf
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 22, 2026
bb33309
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 23, 2026
27946a9
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 23, 2026
0e6ec24
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 24, 2026
27e66d9
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 25, 2026
71ec9f5
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 26, 2026
2532764
fix: key encrypted ABFS reads by the password-free authority
dwsmith1983 Oct 26, 2026
c487ef8
fix: mask overlapping secrets as one span in ABFS diagnostics
dwsmith1983 Oct 26, 2026
5c61d9c
perf: reuse the raw ABFS authority when it has no password
dwsmith1983 Oct 26, 2026
6176c20
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 27, 2026
7f85460
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 27, 2026
8c519a9
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 28, 2026
1aa8c51
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 28, 2026
55fb54d
refactor: render the ABFS native fallback reason's location with disp…
dwsmith1983 Oct 28, 2026
1756d73
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 28, 2026
e23cfac
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 29, 2026
5ba2ac7
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 30, 2026
95b81f6
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 30, 2026
0b8687e
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Oct 31, 2026
bc001bf
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Nov 5, 2026
dbcaa8a
Merge remote-tracking branch 'origin/main' into fix/6605-azure-url-re…
dwsmith1983 Nov 7, 2026
79b5b4f
fix: hide short URI passwords in ABFS driver warnings
dwsmith1983 Nov 7, 2026
7c2a120
fix: keep URL passwords and query strings out of rejected path reasons
dwsmith1983 Nov 7, 2026
3cfb22f
Merge remote-tracking branch 'origin/main' into fix/7128-rejected-pat…
dwsmith1983 Nov 7, 2026
e7d8e2f
Merge branch 'main' into fix/7128-rejected-path-reason
dwsmith1983 Nov 9, 2026
ccb84ae
Merge remote-tracking branch 'origin/main' into fix/7128-rejected-pat…
dwsmith1983 Nov 12, 2026
e969000
Merge remote-tracking branch 'origin/main' into fix/7128-rejected-pat…
dwsmith1983 Nov 14, 2026
196823e
Merge remote-tracking branch 'origin/main' into fix/7128-rejected-pat…
dwsmith1983 Nov 15, 2026
e5fd8a6
Merge remote-tracking branch 'origin/main' into fix/7128-rejected-pat…
dwsmith1983 Nov 20, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -554,8 +554,8 @@ object DeltaScanSupport {
* is uncached but is a plain native URL parse with no I/O
* ([[CometScanRule.objectStoreAcceptsPath]]), so callers pass every complete selected path
* (root paths, data files and deletion vectors) once per distinct URI; a converted table can
* carry the rejected character in a file basename. The reason masks any userinfo in the named
* URI ([[redactedAuthority]]).
* carry the rejected character in a file basename. The reason names the URI without its
* password, query or fragment (`NativeConfig.displayPath`).
*/
private[delta] def objectStoreRejectedPathReason(
uris: Seq[URI],
Expand All @@ -567,11 +567,8 @@ object DeltaScanSupport {
!CometScanRule.objectStoreAcceptsPath(uri)
}
.map { uri =>
// Mask userinfo (see redactedAuthority); the raw path keeps its percent encoding so the
// reason shows the rejected sequence as written.
val shown =
if (uriUserInfo(uri).isEmpty) uri.toString
else s"${redactedAuthority(uri)}${Option(uri.getRawPath).getOrElse("")}"
// The raw path keeps its percent encoding, so the reason shows the rejected sequence.
val shown = NativeConfig.displayPath(uri)
s"Native Delta scan cannot open path '$shown': object_store rejects it " +
"(e.g. an unsupported character in the path)"
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -362,14 +362,24 @@ class DeltaScanContribSuite extends CometDeltaTestBase {
DeltaScanSupport
.objectStoreRejectedPathReason(Seq(new URI("hdfs://nn/dir%0A/data")), Set("hdfs"))
.isEmpty)
// Userinfo is masked in the reason (redactedAuthority's invariant); the path is kept.
// Userinfo is masked in the reason (NativeConfig.displayPath); the path is kept.
val withUserInfo = DeltaScanSupport.objectStoreRejectedPathReason(
Seq(new URI("s3a://key:secret@bucket/dir%0A/data")),
Set("hdfs"))
assert(withUserInfo.exists(_.contains("'s3a://***@bucket/dir%0A/data'")), s"$withUserInfo")
assert(!withUserInfo.exists(_.contains("secret")))
}

test("objectStoreRejectedPathReason drops the query and fragment of the named path") {
// The `%0A` is what makes object_store reject the path; the query is where a SAS token or a
// presigned signature would be.
val uri = new URI("file:///tmp/dir%0A/data?sig=SAS-MARKER#frag")
assert(!CometScanRule.objectStoreAcceptsPath(uri))
val reason = DeltaScanSupport.objectStoreRejectedPathReason(Seq(uri), Set("hdfs"))
assert(reason.exists(_.contains("cannot open path 'file:///tmp/dir%0A/data'")), s"$reason")
Seq("SAS-MARKER", "frag").foreach(marker => assert(!reason.exists(_.contains(marker))))
}

test(
"objectStoreRejectedPathReason declines a selected file whose basename object_store " +
"rejects even though its parent directory is accepted") {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -152,6 +152,31 @@ object NativeConfig {
private[comet] def displayAuthority(uri: URI): String =
s"${lowerScheme(uri).getOrElse("file")}://${authorityWithoutPassword(uri)}"

/**
* `uri` for a message that names a path: no password, query or fragment (where a SAS token or a
* signature would be). Other user info shows as `***@`, except an ABFS user, which is the
* container. The raw path keeps its percent encoding, and a URI with no user info keeps its own
* spelling (`file:/p` and `file:///p` stay apart). A `?` that Hadoop's `Path` encoded as `%3F`
* is part of the path and stays. An opaque URI shows only its scheme.
*/
private[comet] def displayPath(uri: URI): String = {
val rawAuthority = Option(uri.getRawAuthority).getOrElse("")
val at = rawAuthority.lastIndexOf('@')
if (uri.isOpaque) {
s"${uri.getScheme}:"
} else if (at < 0) {
val shown = uri.toString
val end = shown.indexWhere(c => c == '?' || c == '#')
if (end < 0) shown else shown.substring(0, end)
} else {
val scheme = Option(uri.getScheme).map(_ + ":").getOrElse("")
val authority =
if (isAbfs(uri)) authorityWithoutPassword(uri)
else s"***@${rawAuthority.substring(at + 1)}"
s"$scheme//$authority${Option(uri.getRawPath).getOrElse("")}"
}
}

/**
* The schemes native reads through libhdfs: the `fs.comet.libhdfs.schemes` list, or only `hdfs`
* when it is unset or blank, as native `is_hdfs_scheme` decides.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1627,7 +1627,7 @@ object CometScanRule extends Logging {
root.scheme.isDefined && !root.isLibhdfs && !root.isAlias &&
!objectStoreAcceptsPath(root.uri))
.map(root =>
s"$scanName cannot open path '${root.uri}': object_store " +
s"$scanName cannot open path '${NativeConfig.displayPath(root.uri)}': object_store " +
"rejects it (e.g. an unsupported character in the path)")
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import org.scalatest.funsuite.AnyFunSuite
import org.scalatest.matchers.should.Matchers

import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.Path

import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY}
import org.apache.comet.objectstore.AbfsAuthResolver.{Keys, Markers, Providers}
Expand Down Expand Up @@ -770,6 +771,44 @@ class NativeConfigSuite extends AnyFunSuite with Matchers {
NativeConfig.displayAuthority(new URI("/tmp/t/k.parquet")) shouldBe "file://"
}

test("displayPath - no password, query or fragment, and user info masked outside ABFS") {
val cases = Seq(
"s3a://alice:SECRET@bucket:9000/dir%0A/data?sig=SAS-MARKER#f" ->
"s3a://***@bucket:9000/dir%0A/data",
"s3a://alice@bucket/k.parquet" -> "s3a://***@bucket/k.parquet",
// The ABFS user is the container, which names the store, so it stays.
"abfss://data:SECRET@acct.dfs.core.windows.net/p%0A?sig=SAS-MARKER#f" ->
"abfss://data@acct.dfs.core.windows.net/p%0A",
"file:///tmp/dir%0A/data?sig=SAS-MARKER#f" -> "file:///tmp/dir%0A/data",
"s3a://alice:SECRET@[::1]:9000/p?sig=SAS-MARKER" -> "s3a://***@[::1]:9000/p",
// Opaque: there is no authority or path to tell apart from a secret.
"s3a:alice:SECRET@bucket/p?sig=SAS-MARKER" -> "s3a:")
for ((raw, shown) <- cases) {
val display = NativeConfig.displayPath(new URI(raw))
Seq("alice", "SECRET", "SAS-MARKER").foreach(marker => display should not include marker)
display shouldBe shown
}

// An underscore makes the authority registry-based: no parsed user info, but the raw
// authority still holds it.
val registryBased = new URI("s3a://alice:SECRET@my_bucket/p?sig=SAS-MARKER")
registryBased.getRawUserInfo shouldBe null
NativeConfig.displayPath(registryBased) shouldBe "s3a://***@my_bucket/p"

// Nothing to hide: the URI reads as before, spelling included.
val plain = Seq(
new URI("file:///tmp/dir%0A/data"),
new URI("hdfs://nn:8020/t"),
new URI("abfss://data@acct.dfs.core.windows.net/p"),
new Path(new Path("file:/tmp/table"), "part-00000\n.snappy.parquet").toUri)
plain.foreach(uri => NativeConfig.displayPath(uri) shouldBe uri.toString)
NativeConfig.displayPath(plain.last) shouldBe "file:/tmp/table/part-00000%0A.snappy.parquet"

// Hadoop's Path has no query: it encodes `?` into the path, which is kept like any path.
NativeConfig.displayPath(new Path("s3a://bucket/dir\n/data?sig=X").toUri) shouldBe
"s3a://bucket/dir%0A/data%3Fsig=X"
}

test("extractScanObjectStoreOptions - files in one container keep the first file's options") {
val hadoopConf = new Configuration()
hadoopConf.set("fs.azure.account.key.accta.dfs.core.windows.net", "a2V5LWE=")
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -468,4 +468,25 @@ class CometMultiStoreScanSuite extends CometTestBase with AdaptiveSparkPlanHelpe
assert(!options.keys.exists(_.startsWith("fs.azure.account.key.")), options)
}
}

test("a root path object_store rejects is named without the password or a non-ABFS user") {
// The newline (`%0A` in the URI) is what makes object_store reject the root path.
val dir = s"${rootDir.getAbsolutePath}/rejected\n/data"
writeIds(s"file://$dir", 0)
val cases = Seq(
s"s3a://alice:SECRET@bucket$dir" -> "'s3a://***@bucket",
// The ABFS user is the container, which names the store.
s"abfss://data:SECRET@accta.dfs.core.windows.net$dir" ->
"'abfss://data@accta.dfs.core.windows.net")
withSQLConf(nativeScan ++ onePartition: _*) {
for ((path, authority) <- cases) {
val reason = s"cannot open path $authority${rootDir.getAbsolutePath}/rejected%0A/data'"
val (_, cometPlan) = checkSparkAnswerAndFallbackReason(spark.read.parquet(path), reason)
val reasons = new ExtendedExplainInfo().getFallbackReasons(cometPlan)
Seq("alice", "SECRET").foreach { marker =>
assert(!reasons.exists(_.contains(marker)), reasons)
}
}
}
}
}