Python Web Scraper Project

"""Fetch and display the title and links from a public web page you are allowed to access."""

from html.parser import HTMLParser
from urllib.error import HTTPError, URLError
from urllib.parse import urljoin, urlparse
from urllib.request import Request, urlopen


class PageSummary(HTMLParser):
    def __init__(self):
        super().__init__()
        self.in_title = False
        self.title_parts = []
        self.links = []

    def handle_starttag(self, tag, attrs):
        attributes = dict(attrs)
        if tag == "title":
            self.in_title = True
        if tag == "a" and attributes.get("href"):
            self.links.append(attributes["href"])

    def handle_endtag(self, tag):
        if tag == "title":
            self.in_title = False

    def handle_data(self, data):
        if self.in_title:
            self.title_parts.append(data.strip())


def fetch_page(url):
    request = Request(url, headers={"User-Agent": "LearningWebScraper/1.0"})
    with urlopen(request, timeout=15) as response:
        content_type = response.headers.get_content_type()
        if content_type != "text/html":
            raise ValueError(f"Expected an HTML page, received {content_type}.")
        return response.read().decode(response.headers.get_content_charset() or "utf-8", errors="replace")


def main():
    print("Web Page Title and Link Collector")
    print("Only use this on public pages you are allowed to access. Respect each site's terms and robots.txt.\n")
    url = input("Public page URL (including https://): ").strip()
    if urlparse(url).scheme not in {"http", "https"}:
        print("Enter a complete http or https URL.")
        return

    try:
        html = fetch_page(url)
    except (HTTPError, URLError, ValueError) as error:
        print(f"Could not fetch the page: {error}")
        return

    summary = PageSummary()
    summary.feed(html)
    title = " ".join(part for part in summary.title_parts if part) or "No title found"
    unique_links = []
    for link in summary.links:
        absolute = urljoin(url, link)
        if absolute not in unique_links:
            unique_links.append(absolute)

    print(f"\nPage title: {title}")
    print(f"Links found: {len(unique_links)}")
    for link in unique_links[:10]:
        print(f"- {link}")
    if len(unique_links) > 10:
        print("- ...")


if __name__ == "__main__":
    main()