match unsafe element names by local name and namespace, not name()

cssselect/xpath.py

@@ -423,20 +423,30 @@ def xpath_attrib(self, selector: Attrib) -> XPathExpr:
             name = ascii_lower(selector.attrib)
         else:
             name = selector.attrib
+        safe = is_safe_name(name)
+        literal = self.xpath_literal(name)
         if selector.namespace == "*":
             # Namespace wildcard, e.g. "[*|href]": any namespace, including
             # none. XPath 1.0 has no "*:name" name test for this, so match
             # by local name instead.
-            attrib = f"attribute::*[local-name() = {self.xpath_literal(name)}]"
+            attrib = f"attribute::*[local-name() = {literal}]"
+        elif not selector.namespace and safe:
+            attrib = "@" + name
+        elif not selector.namespace:
+            # Like for elements, a name unusable in an XPath name test is
+            # matched by local name and the lack of a namespace.
+            attrib = f"attribute::*[local-name() = {literal} and namespace-uri() = '']"
+        elif not is_safe_name(selector.namespace):
+            # A prefix unusable in an XPath name test cannot be resolved
+            # through the prefix mapping: compare the serialized name instead.
+            qualified = f"{selector.namespace}:{name}"
+            attrib = f"attribute::*[name() = {self.xpath_literal(qualified)}]"
+        elif safe:
+            attrib = f"@{selector.namespace}:{name}"
         else:
-            safe = is_safe_name(name)
-            if selector.namespace:
-                name = f"{selector.namespace}:{name}"
-                safe = safe and is_safe_name(selector.namespace)
-            if safe:
-                attrib = "@" + name
-            else:
-                attrib = f"attribute::*[name() = {self.xpath_literal(name)}]"
+            # Test the local name within the mapped namespace: name() would
+            # compare the prefix as written in the document instead.
+            attrib = f"attribute::{selector.namespace}:*[local-name() = {literal}]"
         if selector.value is None:
             value = None
         elif self.lower_case_attribute_values:
@@ -494,19 +504,27 @@ def xpath_element(self, selector: Element) -> XPathExpr:
                 xpath = self.xpathexpr_cls(element="*")
                 xpath.add_condition(f"local-name() = {self.xpath_literal(element)}")
                 return xpath
+        prefix = ""
         if selector.namespace and selector.namespace != "*":
             # Namespace prefixes are case-sensitive.
             # http://www.w3.org/TR/css3-namespace/#prefixes
-            element = f"{selector.namespace}:{element}"
-            safe = safe and bool(is_safe_name(selector.namespace))
-        xpath = self.xpathexpr_cls(element=element)
-        if not safe:
-            # Not usable as an XPath name test (e.g. an escaped identifier
-            # like di\a0 v): compare the serialized name instead. Done here
-            # rather than through add_name_test(), which would mistake a ":"
-            # inside such a name for a namespace prefix separator.
-            xpath.add_condition(f"name() = {self.xpath_literal(element)}")
-            xpath.element = "*"
+            prefix = f"{selector.namespace}:"
+            if not is_safe_name(selector.namespace):
+                # A prefix not usable in an XPath name test cannot be
+                # resolved through the prefix mapping: compare the
+                # serialized name instead.
+                xpath = self.xpathexpr_cls(element="*")
+                xpath.add_condition(f"name() = {self.xpath_literal(prefix + element)}")
+                return xpath
+        if safe:
+            return self.xpathexpr_cls(element=prefix + element)
+        # A name unusable in an XPath name test (e.g. di\a0 v) is matched by
+        # local name within the prefix, or by the lack of a namespace.
+        xpath = self.xpathexpr_cls(element=f"{prefix}*")
+        condition = f"local-name() = {self.xpath_literal(element)}"
+        if not prefix:
+            condition += " and namespace-uri() = ''"
+        xpath.add_condition(condition)
         return xpath
 
     # CombinedSelector: dispatch by combinator

tests/test_cssselect.py

@@ -992,21 +992,41 @@ def xpath(css: str) -> str:
         )
 
         # Invalid characters in XPath element names
-        assert xpath(r"di\a0 v") == ("*[name() = 'di v']")  # di\xa0v
-        assert xpath(r"di\[v") == ("*[name() = 'di[v']")
-        assert xpath(r"[h\a0 ref]") == ("*[attribute::*[name() = 'h ref']]")  # h\xa0ref
-        assert xpath(r"[h\]ref]") == ("*[attribute::*[name() = 'h]ref']]")
+        assert xpath(r"di\a0 v") == (  # di\xa0v
+            "*[local-name() = 'di v' and namespace-uri() = '']"
+        )
+        assert xpath(r"di\[v") == ("*[local-name() = 'di[v' and namespace-uri() = '']")
+        assert xpath(r"ns|di\[v") == ("ns:*[local-name() = 'di[v']")
+        assert xpath(r"[ns|h\]ref]") == ("*[attribute::ns:*[local-name() = 'h]ref']]")
+        assert xpath(r"[h\a0 ref]") == (  # h\xa0ref
+            "*[attribute::*[local-name() = 'h ref' and namespace-uri() = '']]"
+        )
+        assert xpath(r"[h\]ref]") == (
+            "*[attribute::*[local-name() = 'h]ref' and namespace-uri() = '']]"
+        )
+        # An escaped ":" is part of the name, not a prefix: no match for a
+        # namespaced p:ref attribute either.
+        assert xpath(r"[p\:ref]") == (
+            "*[attribute::*[local-name() = 'p:ref' and namespace-uri() = '']]"
+        )
         # A trailing newline is not a name character either
-        assert xpath(r"div\a ") == "*[name() = 'div\n']"
-        assert xpath(r"[href\a ]") == "*[attribute::*[name() = 'href\n']]"
+        assert xpath(r"div\a ") == "*[local-name() = 'div\n' and namespace-uri() = '']"
+        assert xpath(r"[href\a ]") == (
+            "*[attribute::*[local-name() = 'href\n' and namespace-uri() = '']]"
+        )
         assert xpath(r"ns\a |div") == "*[name() = 'ns\n:div']"
+        assert xpath(r"[ns\a |href]") == "*[attribute::*[name() = 'ns\n:href']]"
         # An escaped "*" is a name, not the universal selector or wildcard
-        assert xpath(r"\*") == "*[name() = '*']"
-        assert xpath(r"\2a") == "*[name() = '*']"
-        assert xpath(r"e > \*") == "e/*[name() = '*']"
-        assert xpath(r"[\*]") == "*[attribute::*[name() = '*']]"
+        assert xpath(r"\*") == "*[local-name() = '*' and namespace-uri() = '']"
+        assert xpath(r"\2a") == "*[local-name() = '*' and namespace-uri() = '']"
+        assert xpath(r"e > \*") == "e/*[local-name() = '*' and namespace-uri() = '']"
+        assert xpath(r"[\*]") == (
+            "*[attribute::*[local-name() = '*' and namespace-uri() = '']]"
+        )
         # Escaped identifiers survive inside :has() arguments
-        assert xpath(r"e:has(di\[v)") == "e[descendant::*[name() = 'di[v']]"
+        assert xpath(r"e:has(di\[v)") == (
+            "e[descendant::*[local-name() = 'di[v' and namespace-uri() = '']]"
+        )
 
         # :scope, alone and in a compound selector
         assert xpath(":scope") == "*[position() = 1]"
@@ -1109,7 +1129,7 @@ class LowerValues(GenericTranslator):
     def test_add_name_test(self) -> None:
         # Directly exercise XPathExpr.add_name_test(), part of the
         # customization API: translation never feeds it a name unusable in
-        # a node test (xpath_element() compares those with name() itself),
+        # a node test (xpath_element() tests those by local name itself),
         # but a subclass can, and then the name() fallback must be used.
         def name_test(element: str) -> str:
             xpath = XPathExpr(element=element)
@@ -1173,7 +1193,9 @@ def test_unicode_escapes(self) -> None:
             """descendant-or-self::*[@aval = "'  '"]"""
         )
         # A code point beyond the Unicode range is replaced with U+FFFD.
-        assert css_to_xpath(r"\110000") == ("descendant-or-self::*[name() = '\ufffd']")
+        assert css_to_xpath(r"\110000") == (
+            "descendant-or-self::*[local-name() = '\ufffd' and namespace-uri() = '']"
+        )
         # A null code point is likewise replaced with U+FFFD, so no NUL
         # character can leak into the generated XPath string.
         assert css_to_xpath(r"*[aval='\0 ']") == (
@@ -1449,27 +1471,27 @@ def argument_types(css: str) -> list[str]:
 
     def test_html_non_ascii_element_names(self) -> None:
         xpath = HTMLTranslator().css_to_xpath
-        assert xpath("x-Ä") == "descendant-or-self::*[name() = 'x-Ä']"
-        assert xpath("X-Ä") == "descendant-or-self::*[name() = 'x-Ä']"
+        xpath_x_a = (
+            "descendant-or-self::*[local-name() = 'x-Ä' and namespace-uri() = '']"
+        )
+        assert xpath("x-Ä") == xpath_x_a
+        assert xpath("X-Ä") == xpath_x_a
         assert xpath("*|X-Ä") == "descendant-or-self::*[local-name() = 'x-Ä']"
-        assert xpath("X-ä") == "descendant-or-self::*[name() = 'x-ä']"
+        assert xpath("X-ä") == (
+            "descendant-or-self::*[local-name() = 'x-ä' and namespace-uri() = '']"
+        )
 
     def test_html_non_ascii_attribute_names(self) -> None:
         xpath = HTMLTranslator().css_to_xpath
-        assert (
-            xpath("[DATA-Ä]")
-            == "descendant-or-self::*[attribute::*[name() = 'data-Ä']]"
-        )
-        assert (
-            xpath('[DATA-Ä="yes"]')
-            == "descendant-or-self::*[attribute::*[name() = 'data-Ä'] = 'yes']"
-        )
+        data_a = "attribute::*[local-name() = 'data-Ä' and namespace-uri() = '']"
+        assert xpath("[DATA-Ä]") == f"descendant-or-self::*[{data_a}]"
+        assert xpath('[DATA-Ä="yes"]') == f"descendant-or-self::*[{data_a} = 'yes']"
         assert xpath("[*|DATA-Ä]") == (
             "descendant-or-self::*[attribute::*[local-name() = 'data-Ä']]"
         )
-        assert (
-            xpath('[DATA-ä="yes"]')
-            == "descendant-or-self::*[attribute::*[name() = 'data-ä'] = 'yes']"
+        assert xpath('[DATA-ä="yes"]') == (
+            "descendant-or-self::*"
+            "[attribute::*[local-name() = 'data-ä' and namespace-uri() = ''] = 'yes']"
         )
 
     def test_html_non_ascii_lang(self) -> None:
@@ -1864,6 +1886,20 @@ def pcss(css: str) -> list[str]:
         assert pcss("*:not(ns|a)") == ["nil", "plain", "ns-el2"]
         assert pcss("b + ns|c") == ["ns-el2"]
 
+        # Names unusable in an XPath node test (here non-ASCII) must go
+        # through the namespace mapping too, not compare name() as a string.
+        document = etree.XML(
+            '<r xmlns:n="urn:x"><n:ä id="ns-el" n:ö="1"/><ä id="plain" ö="1"/></r>'
+        )
+        assert pcss("ns|ä") == ["ns-el"]
+        assert pcss("ä") == ["plain"]
+        assert pcss("*:not(ns|ä)") == ["nil", "plain"]
+        assert pcss("[ns|ö]") == ["ns-el"]
+        assert pcss("[ö]") == ["plain"]
+        # An escaped ":" is part of the attribute name, not a prefix, so this
+        # must not match the namespaced n:ö attribute by its name().
+        assert pcss(r"[n\:ö]") == []
+
     def test_select_with_wildcard_attribute_values(self) -> None:
         cases = [
             ("=", "prefix-token", "prefix-token"),
@@ -1942,7 +1978,9 @@ def test_select_with_default_namespace(self) -> None:
         # elements in a default namespace. Predicate positions (:is(),
         # :not(), "+") must agree with that, not compare name() (which
         # would match such elements by their unprefixed qualified name).
-        document = etree.XML('<r xmlns="urn:d"><x id="x"/><f id="dns-f"/></r>')
+        document = etree.XML(
+            '<r xmlns="urn:d"><x id="x"/><f id="dns-f"/><ä id="dns-ä"/></r>'
+        )
         css_to_xpath = GenericTranslator().css_to_xpath
 
         def pcss(css: str) -> list[str]:
@@ -1956,7 +1994,13 @@ def pcss(css: str) -> list[str]:
         assert pcss("x + f") == []
         # ... and since "f" does not match the default-namespaced <f>,
         # :not(f) must not exclude it.
-        assert pcss("*:not(f)") == ["nil", "x", "dns-f"]
+        assert pcss("*:not(f)") == ["nil", "x", "dns-f", "dns-ä"]
+        # A name unusable in an XPath node test (here non-ASCII) must follow
+        # the same rule, not match the unprefixed name() of such an element.
+        assert pcss("ä") == []
+        assert pcss(":is(ä)") == []
+        assert pcss("x + ä") == []
+        assert pcss("*:not(ä)") == ["nil", "x", "dns-f", "dns-ä"]
 
     def test_select_shakespeare(self) -> None:
         document = html.document_fromstring(HTML_SHAKESPEARE)