Top 5 Python Web Scraping Libraries in 2025

from selenium import webdriver
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
import time

driver = webdriver.Chrome() # Make sure ChromeDriver is installed
driver.get("https://example.com")

load_more_button = driver.find_element(By.ID, "load-more-btn")
load_more_button.click()

# Wait for new content to load
time.sleep(3)

items = driver.find_elements(By.CSS_SELECTOR, ".item-class")
for item in items:
    print(item.text)

driver.quit()

import requests

response = requests.get("https://www.example.com")
if response.status_code == 200:
    print(response.text)
else:
    print("Failed to retrieve the page.")

import requests
from bs4 import BeautifulSoup

url = 'https://example.com/products' 
response = requests.get(url)

soup = BeautifulSoup(response.text, 'html.parser')

# Find all product containers (assuming each product is within a <div class="product-item">)
products = soup.find_all('div', class_='product-item')

# Loop through each product and extract relevant information
for product in products:
    # Extract the product name (assuming it is within an <h2 class="product-name">)
    product_name = product.find('h2', class_='product-name').text.strip()
    
    # Extract the product price (assuming it is within a <span class="product-price">)
    product_price = product.find('span', class_='product-price').text.strip()
    
    # Extract the product description (assuming it is within a <p class="product-description">)
    product_description = product.find('p', class_='product-description').text.strip()
    
    # Print the extracted data
    print(f'Product Name: {product_name}')
    print(f'Price: {product_price}')
    print(f'Description: {product_description}')
    print('----')

from playwright.sync_api import sync_playwright

def scrape_booking_slots():
    with sync_playwright() as p:
        browser = p.chromium.launch(headless=True) 
        context = browser.new_context()
        page = context.new_page()

        booking_url = "https://example.com/booking"
        page.goto(booking_url)

        # Wait for the target element to load
        page.wait_for_selector(".booking-slots") 

        # Scrape the available slots
        slots = page.query_selector_all(".slot-item")  
        available_slots = []

        for slot in slots:
            time = slot.query_selector(".slot-time").inner_text()
            status = slot.query_selector(".slot-status").inner_text()  
            if status.lower() == "available": # Filter for available slots
                available_slots.append(time)

        # Print the results
        print("Available Booking Slots:")
        for slot in available_slots:
            print(f"- {slot}")
            
        browser.close()

scrape_booking_slots()

import scrapy

class ProductSpider(scrapy.Spider):
    name = "products"
    start_urls = ["https://example.com/products"]

    def parse(self, response):
        for product in response.css(".product"):
            yield {
                "name": product.css(".product-name::text").get(),
                "price": product.css(".product-price::text").get(),
            }

[
  {
    "link": "https://www.roborabbit.com/blog/the-ultimate-guide-to-web-scraping-for-beginners-10-tips-for-efficient-data-extraction/",
    "author": "Josephine Loo",
    "date": "September 2024",
    "description": "In this article, we'll cover the basics of web scraping and share tips to help you improve your data extraction process. No matter if you're just starting or looking to improve your web scraping skills, you'll find practical tips to make your web scraping more efficient.",
    "title": "The Ultimate Guide to Web Scraping for Beginners: 10 Tips for Efficient Data Extraction"
  },
  {
    "link": "https://www.roborabbit.com/blog/our-guide-to-roborabbit-element-interaction/",
    "author": "Julianne Youngberg",
    "date": "August 2024",
    "description": "Roborabbit gives you the tools to automate a wide range of user interactions. By understanding how to effectively use these actions, you can create efficient, reusable automations that mimic human behavior and produce consistent, high-quality results.",
    "title": "Our Guide to Roborabbit Element Interaction"
  },
  {
    "link": "https://www.roborabbit.com/blog/how-to-use-puppeteer-in-aws-lambda-for-web-scraping-via-serverless/",
    "author": "Josephine Loo",
    "date": "August 2024",
    "description": "In this article, we’ll explore how AWS Lambda’s serverless architecture can simplify the deployment of your Puppeteer tasks and walk you through the process of setting up Puppeteer in AWS Lambda using the Serverless framework.",
    "title": "How to Use Puppeteer in AWS Lambda for Web Scraping via Serverless"
  },
  {
    "link": "https://www.roborabbit.com/blog/how-to-test-search-bar-functionality-without-code/",
    "author": "Julianne Youngberg",
    "date": "August 2024",
    "description": "By leveraging the power of nocode automation tools like Roborabbit, website owners can efficiently validate functionality, identify issues, and continually optimize the search experience for visitors. Here are our tips for getting started.",
    "title": "How to Test Search Bar Functionality Without Code"
  },
  {
    "link": "https://www.roborabbit.com/blog/how-to-scrape-lists-of-urls-from-a-web-page-using-roborabbit/",
    "author": "Josephine Loo",
    "date": "July 2024",
    "description": "Discover how to efficiently scrape URLs from websites for analyzing a site's linking structure, uncovering potential backlink opportunities, validating links, etc. using Roborabbit in this article.",
    "title": "How to Scrape Lists of URLs from a Web Page Using Roborabbit"
  },
  {
    "link": "https://www.roborabbit.com/blog/how-to-automatically-save-images-with-roborabbit/",
    "author": "Julianne Youngberg",
    "date": "July 2024",
    "description": "Automatic image extraction can supercharge a variety of use cases. With Roborabbit's browser automation capabilities, you have several options to quickly and efficiently save the images you need.",
    "title": "How to Automatically Save Images with Roborabbit"
  },
  {
    "link": "https://www.roborabbit.com/blog/how-to-use-a-proxy-in-curl/",
    "author": "Josephine Loo",
    "date": "July 2024",
    "description": "cURL, known for its simplicity and versatility, is a preferred tool for API testing, web automation, and web scraping. In this article, we’ll learn how to use cURL with a proxy, to facilitate tasks that require enhanced privacy and security, bypass geo-restrictions, or route requests.",
    "title": "How to Use a Proxy in cURL"
  },
  {
    "link": "https://www.roborabbit.com/blog/how-to-debug-your-roborabbit-tasks-a-nocode-guide/",
    "author": "Julianne Youngberg",
    "date": "July 2024",
    "description": "Debugging automated browser tasks can be a challenge, and knowing where to start troubleshooting can be a lifesaver. This guide will show you where to start when a Roborabbit task goes wrong.",
    "title": "How to Debug Your Roborabbit Tasks (A Nocode Guide)"
  },
  {
    "link": "https://www.roborabbit.com/blog/how-to-make-a-discord-bot-to-send-price-change-alerts-automatically/",
    "author": "Josephine Loo",
    "date": "July 2024",
    "description": "In Discord communities, bots like Carl-bot, MEE6, Dyno, and Pancake are essential for improving server functionality and engaging users. Did you know you can also make your own Discord bot tailored to your specified needs? Let’s learn how in this article!",
    "title": "How to Make a Discord Bot to Send Price Change Alerts Automatically"
  }
]