Building a CLI Web Scraper
This guide demonstrates how to combine multiple Python modules to build a robust Command Line Interface (CLI) tool that scrapes a website and saves the data.
Modules Used:
- click: For the command line interface.
- requests: To fetch the HTML.
- beautifulsoup4: To parse the HTML.
- rich: To display a progress spinner and formatted output.
- json: To save the data.
- pathlib: To handle file paths.
The Code
Save this as scraper.py.
import click
import requests
from bs4 import BeautifulSoup
import json
from pathlib import Path
from rich.console import Console
from rich.table import Table
console = Console()
@click.command()
@click.option('--url', prompt='URL to scrape', help='The URL of the website to scrape.')
@click.option('--output', default='data.json', help='Output JSON file path.')
def scrape(url, output):
"""Scrapes all links and titles from a webpage."""
# 1. Fetch the page with a visual spinner
with console.status(f"[bold green]Fetching {url}...") as status:
try:
response = requests.get(url, timeout=10)
response.raise_for_status()
except Exception as e:
console.print(f"[bold red]Error fetching URL:[/bold red] {e}")
return
# 2. Parse the HTML
soup = BeautifulSoup(response.text, 'html.parser')
links = soup.find_all('a')
extracted_data = []
# 3. Extract Data
for link in links:
text = link.get_text(strip=True)
href = link.get('href')
if text and href:
extracted_data.append({"title": text, "url": href})
# 4. Display Summary Table
table = Table(title=f"Found {len(extracted_data)} Links")
table.add_column("Title", style="cyan", no_wrap=True)
table.add_column("URL", style="magenta")
# Show first 5 items in terminal
for item in extracted_data[:5]:
table.add_row(item['title'][:30], item['url'][:50])
console.print(table)
if len(extracted_data) > 5:
console.print(f"...and {len(extracted_data) - 5} more.")
# 5. Save to File
Path(output).write_text(json.dumps(extracted_data, indent=2))
console.print(f"[bold green]Success![/bold green] Data saved to {output}")
if __name__ == '__main__':
scrape()Usage
python scraper.py --url https://news.ycombinator.com --output hacker_news.json