From 32b078bf8ba080498508eed6316aced957613448 Mon Sep 17 00:00:00 2001 From: Glyph Date: Sun, 16 Feb 2020 01:35:08 -0800 Subject: [PATCH 1/8] fix up inconsistencies in parsing & textual representation of 'rooted' and 'uses_netloc' --- src/hyperlink/_url.py | 38 +++++++++++++++++++++++++--------- src/hyperlink/test/test_url.py | 34 ++++++++++++++++++++++++++++++ 2 files changed, 62 insertions(+), 10 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index f39525d3..e20edc4f 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -815,9 +815,9 @@ class URL(object): that starts with a slash. userinfo (Text): The username or colon-separated username:password pair. - uses_netloc (bool): Indicates whether two slashes appear - between the scheme and the host (``http://eg.com`` vs - ``mailto:e@g.com``). Set automatically based on scheme. + uses_netloc (bool): Indicates whether ``://`` will appear to separate + the scheme from the path, even in cases where no host is present. + May be implied by scheme, or set explictly. All of these parts are also exposed as read-only attributes of URL instances, along with several useful methods. @@ -882,15 +882,28 @@ def __init__( self._rooted = _typecheck("rooted", rooted, bool) self._userinfo = _textcheck("userinfo", userinfo, '/?#@') - uses_netloc = scheme_uses_netloc(self._scheme, uses_netloc) + if uses_netloc is None: + uses_netloc = scheme_uses_netloc(self._scheme, uses_netloc) self._uses_netloc = _typecheck("uses_netloc", uses_netloc, bool, NoneType) - # fixup for rooted consistency - if self._host: + will_have_authority = ( + self._host or + (self._port and self._port != SCHEME_PORT_MAP.get(scheme)) + ) + if will_have_authority: + # fixup for rooted consistency; if there's any 'authority' + # represented in the textual URL, then the path must be rooted, and + # we're definitely using a netloc (there must be a ://). self._rooted = True - if (not self._rooted) and self._path and self._path[0] == '': + self._uses_netloc = True + if (not self._rooted) and self.path[:1] == (u'',): self._rooted = True self._path = self._path[1:] + if not will_have_authority and self._path and not self._rooted: + # If, after fixing up the path, there *is* a path and it *isn't* + # rooted, then we are definitely not using a netloc; if we did, it + # would make the path (erroneously) look like a hostname. + self._uses_netloc = False def get_decoded_url(self, lazy=False): # type: (bool) -> DecodedURL @@ -1006,6 +1019,8 @@ def userinfo(self): def uses_netloc(self): # type: () -> Optional[bool] """ + Whether the textual URL representation will contain a ``://`` netloc + separator. """ return self._uses_netloc @@ -1134,14 +1149,17 @@ def replace( slash. userinfo (Text): The username or colon-separated username:password pair. - uses_netloc (bool): Indicates whether two slashes appear between - the scheme and the host - (``http://eg.com`` vs ``mailto:e@g.com``) + uses_netloc (bool): Indicates whether rooted paths should include a + scheme-separator by default. Returns: URL: A copy of the current :class:`URL`, with new values for parameters passed. """ + if scheme is not _UNSET and scheme != self.scheme: + # when changing schemes, reset the explicit uses_netloc preference + # to honor the new scheme. + uses_netloc = None return self.__class__( scheme=_optional(scheme, self.scheme), host=_optional(host, self.host), diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index fa5c7bf1..356def9c 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -817,6 +817,18 @@ def test_mailto(self): self.assertEqual(URL.from_text(u"mailto:user@example.com").to_text(), u"mailto:user@example.com") + def test_httpWithoutHost(self): + # type: () -> None + """ + An HTTP URL without a hostname, but with a path, should also round-trip + cleanly. + """ + without_host = URL.from_text(u"http:relative-path") + self.assertEqual(without_host.host, u'') + self.assertEqual(without_host.path, (u'relative-path',)) + self.assertEqual(without_host.uses_netloc, False) + self.assertEqual(without_host.to_text(), u"http:relative-path") + def test_queryIterable(self): # type: () -> None """ @@ -938,15 +950,25 @@ def test_netloc(self): # type: () -> None url = URL(scheme='https') self.assertEqual(url.uses_netloc, True) + self.assertEqual(url.to_text(), u'https://') + self.assertEqual(URL.from_text('https:').uses_netloc, False) + self.assertEqual(URL.from_text('https://').uses_netloc, True) + + url = URL(scheme='https', uses_netloc=False) + self.assertEqual(url.uses_netloc, False) + self.assertEqual(url.to_text(), u'https:') url = URL(scheme='git+https') self.assertEqual(url.uses_netloc, True) + self.assertEqual(url.to_text(), u'git+https://') url = URL(scheme='mailto') self.assertEqual(url.uses_netloc, False) + self.assertEqual(url.to_text(), u'mailto:') url = URL(scheme='ztp') self.assertEqual(url.uses_netloc, None) + self.assertEqual(url.to_text(), u'ztp:') url = URL.from_text('ztp://test.com') self.assertEqual(url.uses_netloc, True) @@ -1116,6 +1138,18 @@ def test_autorooted(self): self.assertEqual(normal_absolute.rooted, True) self.assertEqual(attempt_unrooted_absolute.rooted, True) + def test_rooted_with_empty_non_none_host(self): + # type: () -> None + """ + The C{rooted} constructor argument will be ignored on URLs that include + a scheme. + """ + directly_constructed = URL(scheme='udp', port=4900) + parsed = URL.from_text('udp://:4900') + self.assertEqual(str(directly_constructed), str(parsed)) + self.assertEqual(directly_constructed.asText(), parsed.asText()) + self.assertEqual(directly_constructed, parsed) + def test_wrong_constructor(self): # type: () -> None with self.assertRaises(ValueError): From 82f096ba9ec018ff746204a634bd27f6468eb1ce Mon Sep 17 00:00:00 2001 From: Glyph Date: Sun, 16 Feb 2020 14:15:53 -0800 Subject: [PATCH 2/8] per CR: make the test a little more thorough, improve docstring --- src/hyperlink/test/test_url.py | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 356def9c..4b96b2eb 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -1138,17 +1138,29 @@ def test_autorooted(self): self.assertEqual(normal_absolute.rooted, True) self.assertEqual(attempt_unrooted_absolute.rooted, True) - def test_rooted_with_empty_non_none_host(self): + def test_rooted_with_port_but_no_host(self): # type: () -> None """ - The C{rooted} constructor argument will be ignored on URLs that include - a scheme. + URLs which include a netloc separator are inherently rooted, regardless + of whether they include one because they specify an explicit host or + port, whether they are parsed or directly constructed, and whether the + ``rooted`` constructor argument is supplied or not. """ - directly_constructed = URL(scheme='udp', port=4900) + directly_constructed = URL(scheme='udp', port=4900, rooted=False) + directly_constructed_implict = URL(scheme='udp', port=4900) + directly_constructed_rooted = URL(scheme=u'udp', port=4900, rooted=True) + self.assertEqual(directly_constructed.rooted, True) + self.assertEqual(directly_constructed_implict.rooted, True) + self.assertEqual(directly_constructed_rooted.rooted, True) parsed = URL.from_text('udp://:4900') self.assertEqual(str(directly_constructed), str(parsed)) + self.assertEqual(str(directly_constructed_implict), str(parsed)) self.assertEqual(directly_constructed.asText(), parsed.asText()) self.assertEqual(directly_constructed, parsed) + self.assertEqual(directly_constructed, directly_constructed_implict) + self.assertEqual(directly_constructed, directly_constructed_rooted) + self.assertEqual(directly_constructed_implict, parsed) + self.assertEqual(directly_constructed_rooted, parsed) def test_wrong_constructor(self): # type: () -> None From b3d03b48c736bb026879d598ef8c380d78c711bc Mon Sep 17 00:00:00 2001 From: Glyph Date: Sun, 16 Feb 2020 14:20:41 -0800 Subject: [PATCH 3/8] <79 --- src/hyperlink/test/test_url.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 4b96b2eb..668ddd84 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -1148,7 +1148,8 @@ def test_rooted_with_port_but_no_host(self): """ directly_constructed = URL(scheme='udp', port=4900, rooted=False) directly_constructed_implict = URL(scheme='udp', port=4900) - directly_constructed_rooted = URL(scheme=u'udp', port=4900, rooted=True) + directly_constructed_rooted = URL(scheme=u'udp', port=4900, + rooted=True) self.assertEqual(directly_constructed.rooted, True) self.assertEqual(directly_constructed_implict.rooted, True) self.assertEqual(directly_constructed_rooted.rooted, True) From a1b360c56d84fcd30dae758c6fc202e700c5435e Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 31 Mar 2020 00:10:05 -0700 Subject: [PATCH 4/8] per CR: explain in much more detail --- src/hyperlink/_url.py | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 8fe0ef7b..51b926cd 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -815,9 +815,18 @@ class URL(object): that starts with a slash. userinfo (Text): The username or colon-separated username:password pair. - uses_netloc (bool): Indicates whether ``://`` will appear to separate - the scheme from the path, even in cases where no host is present. - May be implied by scheme, or set explictly. + uses_netloc (Optional[bool]): Indicates whether ``://`` (the "netloc + separator") will appear to separate the scheme from the *path* in + cases where no host is present. Setting this to ``True`` is a + non-spec-compliant affordance for the common practice of having URIs + that are *not* URLs (cannot have a 'host' part) but nevertheless use + the common ``://`` idiom that most people associate with URLs; + e.g. ``message:`` URIs like ``message://message-id`` being + equivalent to ``message:message-id``. This may be inferred based on + the scheme depending on whether :func:`register_scheme` has been + used to register the scheme and should not be passed directly unless + you know the scheme works like this and you know it has not been + registered. All of these parts are also exposed as read-only attributes of URL instances, along with several useful methods. From 97c62c27a0a500ff7b4c81ebb9784a18b392fb4d Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 31 Mar 2020 00:18:29 -0700 Subject: [PATCH 5/8] match __init__ doc --- src/hyperlink/_url.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index 51b926cd..d8329985 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -1028,8 +1028,8 @@ def userinfo(self): def uses_netloc(self): # type: () -> Optional[bool] """ - Whether the textual URL representation will contain a ``://`` netloc - separator. + Indicates whether ``://`` (the "netloc separator") will appear to + separate the scheme from the *path* in cases where no host is present. """ return self._uses_netloc From 4b81bb5db8a77410649c3f39bce6e08097e9d9cf Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 31 Mar 2020 00:22:14 -0700 Subject: [PATCH 6/8] per CR: match __init__ --- src/hyperlink/_url.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/src/hyperlink/_url.py b/src/hyperlink/_url.py index d8329985..244406f2 100644 --- a/src/hyperlink/_url.py +++ b/src/hyperlink/_url.py @@ -1158,8 +1158,19 @@ def replace( slash. userinfo (Text): The username or colon-separated username:password pair. - uses_netloc (bool): Indicates whether rooted paths should include a - scheme-separator by default. + uses_netloc (bool): Indicates whether ``://`` (the "netloc + separator") will appear to separate the scheme from the *path* + in cases where no host is present. Setting this to ``True`` is + a non-spec-compliant affordance for the common practice of + having URIs that are *not* URLs (cannot have a 'host' part) but + nevertheless use the common ``://`` idiom that most people + associate with URLs; e.g. ``message:`` URIs like + ``message://message-id`` being equivalent to + ``message:message-id``. This may be inferred based on the + scheme depending on whether :func:`register_scheme` has been + used to register the scheme and should not be passed directly + unless you know the scheme works like this and you know it has + not been registered. Returns: URL: A copy of the current :class:`URL`, with new values for From aae83a8b4af091cb5377bab32d643f74d197a586 Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 31 Mar 2020 00:28:06 -0700 Subject: [PATCH 7/8] per CR: add https:/, enumerate the cases --- src/hyperlink/test/test_url.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 668ddd84..12b8ffcf 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -951,7 +951,11 @@ def test_netloc(self): url = URL(scheme='https') self.assertEqual(url.uses_netloc, True) self.assertEqual(url.to_text(), u'https://') + # scheme, no host, no path, no netloc hack self.assertEqual(URL.from_text('https:').uses_netloc, False) + # scheme, no host, absolute path, no netloc hack + self.assertEqual(URL.from_text('https:/').uses_netloc, False) + # scheme, no host, no path, netloc hack to indicate :// syntax self.assertEqual(URL.from_text('https://').uses_netloc, True) url = URL(scheme='https', uses_netloc=False) From a5bc24123a96c04f16067a555aeafa686195f82b Mon Sep 17 00:00:00 2001 From: Glyph Date: Tue, 31 Mar 2020 00:37:00 -0700 Subject: [PATCH 8/8] per CR: rephrase gibberish test docstring --- src/hyperlink/test/test_url.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/hyperlink/test/test_url.py b/src/hyperlink/test/test_url.py index 12b8ffcf..6952e42f 100644 --- a/src/hyperlink/test/test_url.py +++ b/src/hyperlink/test/test_url.py @@ -1145,10 +1145,13 @@ def test_autorooted(self): def test_rooted_with_port_but_no_host(self): # type: () -> None """ - URLs which include a netloc separator are inherently rooted, regardless - of whether they include one because they specify an explicit host or - port, whether they are parsed or directly constructed, and whether the - ``rooted`` constructor argument is supplied or not. + URLs which include a ``://`` netloc-separator for any reason are + inherently rooted, regardless of the value or presence of the + ``rooted`` constructor argument. + + They may include a netloc-separator because their constructor was + directly invoked with an explicit host or port, or because they were + parsed from a string which included the literal ``://`` separator. """ directly_constructed = URL(scheme='udp', port=4900, rooted=False) directly_constructed_implict = URL(scheme='udp', port=4900)