TDM 20200: Project 2 - Web Scraping Introduction 2

import requests
from lxml import html

# Fetch a web page
url = "https://example.com"
response = requests.get(url)

# Check if the request was successful
print(f"Status code: {response.status_code}")
print(f"Content length: {len(response.content)} bytes")

# The HTML content is in response.text
print("\nFirst 500 characters of HTML:")
print(response.text[:500])

# Parse the HTML content
tree = html.fromstring(response.text)

# Extract the title
title = tree.xpath('//title/text()')
print(f"Page title: {title[0] if title else 'Not found'}")

# Extract all paragraph text
paragraphs = tree.xpath('//p/text()')
print(f"\nParagraphs found: {len(paragraphs)}")
for i, para in enumerate(paragraphs, 1):
    print(f"{i}. {para}")

# Extract the heading
heading = tree.xpath('//h1/text()')
print(f"\nHeading: {heading[0] if heading else 'Not found'}")

import requests
from lxml import html

url = "https://quotes.toscrape.com"
response = requests.get(url)
tree = html.fromstring(response.text)

# Find all quote containers (divs with class="quote")
# You can find this by inspecting the page!
quote_containers = tree.xpath('//div[@class="quote"]')

print(f"Found {len(quote_containers)} quote containers\n")

# Extract data from each container
for i, container in enumerate(quote_containers[:3], 1):  # First 3 quotes
    # Extract quote text from within this container
    quote_text = container.xpath('.//span[@class="text"]/text()')[0]

    # Extract author from within this container
    author = container.xpath('.//small[@class="author"]/text()')[0]

    # Extract tags from within this container
    tags = container.xpath('.//a[@class="tag"]/text()')

    print(f"Quote {i}:")
    print(f"  Text: {quote_text}")
    print(f"  Author: {author}")
    print(f"  Tags: {', '.join(tags)}")
    print()

import requests
from lxml import html

url = "https://the-examples-book.com"
response = requests.get(url)

# Always check the status code first
if response.status_code == 200:
    print("Successfully fetched the page!")
    tree = html.fromstring(response.text)

    # Use inspect element to find what you want to extract
    # For example, let's find all links on the page
    links = tree.xpath('//a/@href')
    print(f"\nFound {len(links)} links on the page")

    # Show first 10 links
    for i, link in enumerate(links[:10], 1):
        print(f"{i}. {link}")
else:
    print(f"Error: Status code {response.status_code}")

import requests
from lxml import html
import time
from urllib.parse import urljoin

base_url = "https://quotes.toscrape.com"
current_url = base_url
all_quotes = []
page_num = 1

while True:
    print(f"Scraping page {page_num}...")
    response = requests.get(current_url)
    tree = html.fromstring(response.text)

    # Extract quotes from current page
    quote_containers = tree.xpath('//div[@class="quote"]')
    for container in quote_containers:
        text = container.xpath('.//span[@class="text"]/text()')[0]
        author = container.xpath('.//small[@class="author"]/text()')[0]
        tags = container.xpath('.//a[@class="tag"]/text()')
        all_quotes.append({"text": text, "author": author, "tags": tags})

    # Check for "Next" button using inspect element to find the selector
    next_button = tree.xpath('//li[@class="next"]/a/@href')

    if next_button:
        # Construct full URL (handling relative URLs)
        next_url = next_button[0]
        # Use urljoin to properly combine URLs
        current_url = urljoin(base_url, next_url)
        page_num += 1
        time.sleep(1)  # Be polite - wait between requests
    else:
        print("No more pages!")
        break

print(f"\nTotal quotes scraped: {len(all_quotes)}")
print(f"\nFirst 3 quotes:")
for i, quote in enumerate(all_quotes[:3], 1):
    print(f"{i}. {quote['text']} - {quote['author']} - Tags: {', '.join(quote['tags'])}")

import requests
from lxml import html
from urllib.parse import urljoin

url = "https://quotes.toscrape.com"
response = requests.get(url)
tree = html.fromstring(response.text)

# Extract author profile links
author_link_elements = tree.xpath('//a[contains(@href, "/author/")]')
author_dict = {}

for elem in author_link_elements:
    author_name = elem.text
    author_href = elem.get('href')
    # Convert relative URL to absolute
    full_url = urljoin(url, author_href)
    author_dict[author_name] = full_url

print("Author name to profile URL mapping:")
for name, profile_url in list(author_dict.items())[:5]:
    print(f"{name}: {profile_url}")

# Extract tag links
tag_elements = tree.xpath('//a[@class="tag"]')
print(f"\nFound {len(tag_elements)} tag links")
print("\nFirst 10 tags:")
for i, tag_elem in enumerate(tag_elements[:10], 1):
    tag_text = tag_elem.text
    tag_href = tag_elem.get('href')
    tag_url = urljoin(url, tag_href)
    print(f"{i}. {tag_text} -> {tag_url}")