from urllib.parse import ParseResult, urlparse, urlsplit
def parse(domain: str, scheme: str = 'http') -> str:
"""
Convert domain name to 'http://www.domain.com' format
"""
# https://stackoverflow.com/questions/21659044/how-can-i-prepend-http-to-a-url-if-it-doesnt-begin-with-http
p = urlparse(domain, 'http')
netloc = p.netloc or p.path
path = p.path if p.netloc else ''
p = ParseResult(p.scheme, netloc, path, *p[3:])
return p.geturl()
def hostname_only(value: str) -> str:
"""
Reduce a domain or a URL to its bare, lowercased hostname.
The canonical host-comparison helper: report.py matches
ScrapeResult.website against the domain list with it, and
ConfigManager.is_domain_ignored() normalises with it before applying
ignore_list patterns. Both receive a mix of shapes -- get_domains() runs
every domain through parse(), so it yields 'http://host', while
ScrapeResult.website holds the post-redirect URL (scheme + trailing slash
+ path, sometimes a port).
Implemented on urlsplit() of a '//'-prefixed value rather than
urlparse(value, 'http'): for a scheme-less 'host:port' the RFC 3986
scheme grammar permits dots, so urlparse reads 'sub.example.com' as the
scheme and '8080' as the path. Forcing netloc parsing also gets port,
credential and IPv6-bracket stripping from .hostname for free.
Never raises. Returns '' for input with no host, and for input urlsplit()
rejects -- notably a bracketed fnmatch character class such as
'*.test[0-9].example.com', which it reads as a malformed IPv6 literal.
ignore_list entries may legitimately contain those, and a caller treats ''
as "not a hostname", falling back to verbatim matching. An exception here
would instead escape is_domain_ignored() and abort the whole scanner
iteration plus every report path.
"""
value = (value or '').strip()
if '://' not in value and not value.startswith('//'):
value = f'//{value}'
try:
return urlsplit(value).hostname or ''
except ValueError:
return ''
|