diff --git a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala index 15684b47c06..e2112bda003 100644 --- a/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala +++ b/contrib/delta-spark/src/main/scala/org/apache/comet/contrib/delta/DeltaScanSupport.scala @@ -554,8 +554,8 @@ object DeltaScanSupport { * is uncached but is a plain native URL parse with no I/O * ([[CometScanRule.objectStoreAcceptsPath]]), so callers pass every complete selected path * (root paths, data files and deletion vectors) once per distinct URI; a converted table can - * carry the rejected character in a file basename. The reason masks any userinfo in the named - * URI ([[redactedAuthority]]). + * carry the rejected character in a file basename. The reason names the URI without its + * password, query or fragment (`NativeConfig.displayPath`). */ private[delta] def objectStoreRejectedPathReason( uris: Seq[URI], @@ -567,11 +567,8 @@ object DeltaScanSupport { !CometScanRule.objectStoreAcceptsPath(uri) } .map { uri => - // Mask userinfo (see redactedAuthority); the raw path keeps its percent encoding so the - // reason shows the rejected sequence as written. - val shown = - if (uriUserInfo(uri).isEmpty) uri.toString - else s"${redactedAuthority(uri)}${Option(uri.getRawPath).getOrElse("")}" + // The raw path keeps its percent encoding, so the reason shows the rejected sequence. + val shown = NativeConfig.displayPath(uri) s"Native Delta scan cannot open path '$shown': object_store rejects it " + "(e.g. an unsupported character in the path)" } diff --git a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala index 1d240ba3e22..28f95e0d7e1 100644 --- a/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala +++ b/contrib/delta-spark/src/test/scala/org/apache/comet/contrib/delta/DeltaScanContribSuite.scala @@ -362,7 +362,7 @@ class DeltaScanContribSuite extends CometDeltaTestBase { DeltaScanSupport .objectStoreRejectedPathReason(Seq(new URI("hdfs://nn/dir%0A/data")), Set("hdfs")) .isEmpty) - // Userinfo is masked in the reason (redactedAuthority's invariant); the path is kept. + // Userinfo is masked in the reason (NativeConfig.displayPath); the path is kept. val withUserInfo = DeltaScanSupport.objectStoreRejectedPathReason( Seq(new URI("s3a://key:secret@bucket/dir%0A/data")), Set("hdfs")) @@ -370,6 +370,16 @@ class DeltaScanContribSuite extends CometDeltaTestBase { assert(!withUserInfo.exists(_.contains("secret"))) } + test("objectStoreRejectedPathReason drops the query and fragment of the named path") { + // The `%0A` is what makes object_store reject the path; the query is where a SAS token or a + // presigned signature would be. + val uri = new URI("file:///tmp/dir%0A/data?sig=SAS-MARKER#frag") + assert(!CometScanRule.objectStoreAcceptsPath(uri)) + val reason = DeltaScanSupport.objectStoreRejectedPathReason(Seq(uri), Set("hdfs")) + assert(reason.exists(_.contains("cannot open path 'file:///tmp/dir%0A/data'")), s"$reason") + Seq("SAS-MARKER", "frag").foreach(marker => assert(!reason.exists(_.contains(marker)))) + } + test( "objectStoreRejectedPathReason declines a selected file whose basename object_store " + "rejects even though its parent directory is accepted") { diff --git a/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala b/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala index b4b5bf95b39..93fc8cd501f 100644 --- a/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala +++ b/spark/src/main/scala/org/apache/comet/objectstore/NativeConfig.scala @@ -152,6 +152,31 @@ object NativeConfig { private[comet] def displayAuthority(uri: URI): String = s"${lowerScheme(uri).getOrElse("file")}://${authorityWithoutPassword(uri)}" + /** + * `uri` for a message that names a path: no password, query or fragment (where a SAS token or a + * signature would be). Other user info shows as `***@`, except an ABFS user, which is the + * container. The raw path keeps its percent encoding, and a URI with no user info keeps its own + * spelling (`file:/p` and `file:///p` stay apart). A `?` that Hadoop's `Path` encoded as `%3F` + * is part of the path and stays. An opaque URI shows only its scheme. + */ + private[comet] def displayPath(uri: URI): String = { + val rawAuthority = Option(uri.getRawAuthority).getOrElse("") + val at = rawAuthority.lastIndexOf('@') + if (uri.isOpaque) { + s"${uri.getScheme}:" + } else if (at < 0) { + val shown = uri.toString + val end = shown.indexWhere(c => c == '?' || c == '#') + if (end < 0) shown else shown.substring(0, end) + } else { + val scheme = Option(uri.getScheme).map(_ + ":").getOrElse("") + val authority = + if (isAbfs(uri)) authorityWithoutPassword(uri) + else s"***@${rawAuthority.substring(at + 1)}" + s"$scheme//$authority${Option(uri.getRawPath).getOrElse("")}" + } + } + /** * The schemes native reads through libhdfs: the `fs.comet.libhdfs.schemes` list, or only `hdfs` * when it is unset or blank, as native `is_hdfs_scheme` decides. diff --git a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala index 722ec1a4402..e436fccd9fc 100644 --- a/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala +++ b/spark/src/main/scala/org/apache/comet/rules/CometScanRule.scala @@ -1627,7 +1627,7 @@ object CometScanRule extends Logging { root.scheme.isDefined && !root.isLibhdfs && !root.isAlias && !objectStoreAcceptsPath(root.uri)) .map(root => - s"$scanName cannot open path '${root.uri}': object_store " + + s"$scanName cannot open path '${NativeConfig.displayPath(root.uri)}': object_store " + "rejects it (e.g. an unsupported character in the path)") } diff --git a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala index 34d885aa319..3de97672ca8 100644 --- a/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala +++ b/spark/src/test/scala/org/apache/comet/objectstore/NativeConfigSuite.scala @@ -25,6 +25,7 @@ import org.scalatest.funsuite.AnyFunSuite import org.scalatest.matchers.should.Matchers import org.apache.hadoop.conf.Configuration +import org.apache.hadoop.fs.Path import org.apache.comet.CometConf.{COMET_LIBHDFS_SCHEMES_KEY, COMET_S3_COMPLIANT_SCHEMES_KEY} import org.apache.comet.objectstore.AbfsAuthResolver.{Keys, Markers, Providers} @@ -770,6 +771,44 @@ class NativeConfigSuite extends AnyFunSuite with Matchers { NativeConfig.displayAuthority(new URI("/tmp/t/k.parquet")) shouldBe "file://" } + test("displayPath - no password, query or fragment, and user info masked outside ABFS") { + val cases = Seq( + "s3a://alice:SECRET@bucket:9000/dir%0A/data?sig=SAS-MARKER#f" -> + "s3a://***@bucket:9000/dir%0A/data", + "s3a://alice@bucket/k.parquet" -> "s3a://***@bucket/k.parquet", + // The ABFS user is the container, which names the store, so it stays. + "abfss://data:SECRET@acct.dfs.core.windows.net/p%0A?sig=SAS-MARKER#f" -> + "abfss://data@acct.dfs.core.windows.net/p%0A", + "file:///tmp/dir%0A/data?sig=SAS-MARKER#f" -> "file:///tmp/dir%0A/data", + "s3a://alice:SECRET@[::1]:9000/p?sig=SAS-MARKER" -> "s3a://***@[::1]:9000/p", + // Opaque: there is no authority or path to tell apart from a secret. + "s3a:alice:SECRET@bucket/p?sig=SAS-MARKER" -> "s3a:") + for ((raw, shown) <- cases) { + val display = NativeConfig.displayPath(new URI(raw)) + Seq("alice", "SECRET", "SAS-MARKER").foreach(marker => display should not include marker) + display shouldBe shown + } + + // An underscore makes the authority registry-based: no parsed user info, but the raw + // authority still holds it. + val registryBased = new URI("s3a://alice:SECRET@my_bucket/p?sig=SAS-MARKER") + registryBased.getRawUserInfo shouldBe null + NativeConfig.displayPath(registryBased) shouldBe "s3a://***@my_bucket/p" + + // Nothing to hide: the URI reads as before, spelling included. + val plain = Seq( + new URI("file:///tmp/dir%0A/data"), + new URI("hdfs://nn:8020/t"), + new URI("abfss://data@acct.dfs.core.windows.net/p"), + new Path(new Path("file:/tmp/table"), "part-00000\n.snappy.parquet").toUri) + plain.foreach(uri => NativeConfig.displayPath(uri) shouldBe uri.toString) + NativeConfig.displayPath(plain.last) shouldBe "file:/tmp/table/part-00000%0A.snappy.parquet" + + // Hadoop's Path has no query: it encodes `?` into the path, which is kept like any path. + NativeConfig.displayPath(new Path("s3a://bucket/dir\n/data?sig=X").toUri) shouldBe + "s3a://bucket/dir%0A/data%3Fsig=X" + } + test("extractScanObjectStoreOptions - files in one container keep the first file's options") { val hadoopConf = new Configuration() hadoopConf.set("fs.azure.account.key.accta.dfs.core.windows.net", "a2V5LWE=") diff --git a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala index a650ebcf2f5..0abe24bbbd8 100644 --- a/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala +++ b/spark/src/test/scala/org/apache/comet/rules/CometMultiStoreScanSuite.scala @@ -468,4 +468,25 @@ class CometMultiStoreScanSuite extends CometTestBase with AdaptiveSparkPlanHelpe assert(!options.keys.exists(_.startsWith("fs.azure.account.key.")), options) } } + + test("a root path object_store rejects is named without the password or a non-ABFS user") { + // The newline (`%0A` in the URI) is what makes object_store reject the root path. + val dir = s"${rootDir.getAbsolutePath}/rejected\n/data" + writeIds(s"file://$dir", 0) + val cases = Seq( + s"s3a://alice:SECRET@bucket$dir" -> "'s3a://***@bucket", + // The ABFS user is the container, which names the store. + s"abfss://data:SECRET@accta.dfs.core.windows.net$dir" -> + "'abfss://data@accta.dfs.core.windows.net") + withSQLConf(nativeScan ++ onePartition: _*) { + for ((path, authority) <- cases) { + val reason = s"cannot open path $authority${rootDir.getAbsolutePath}/rejected%0A/data'" + val (_, cometPlan) = checkSparkAnswerAndFallbackReason(spark.read.parquet(path), reason) + val reasons = new ExtendedExplainInfo().getFallbackReasons(cometPlan) + Seq("alice", "SECRET").foreach { marker => + assert(!reasons.exists(_.contains(marker)), reasons) + } + } + } + } }