Build a Python Amazon Scraper That Bypasses Blocks Using Proxies and Best Practices

import requests
from bs4 import BeautifulSoup

# Target URL
url = 'https://books.toscrape.com/'

# Set a user agent to mimic a browser
headers = {
    'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
}

# Send GET request
response = requests.get(url, headers=headers)

# Check if request was successful
if response.status_code == 200:
    # Parse HTML content
    soup = BeautifulSoup(response.text, 'html.parser')
    
    # Find all book containers
    books = soup.select('article.product_pod')
    
    # Extract and print book titles
    for book in books[:5]:  # First 5 books
        title = book.h3.a['title']
        price = book.select_one('p.price_color').text
        print(f'Title: {title}\nPrice: {price}\n')
else:
    print(f'Failed to retrieve the webpage: {response.status_code}')

import requests
from bs4 import BeautifulSoup
import time

def fetch(url, proxy=None, max_retries=3):
    """Fetch URL with retry logic and proxy support"""
    headers = {
        'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36'
    }
    
    proxies = None
    if proxy:
        proxies = {
            'http': f'http://{proxy}',
            'https': f'http://{proxy}'
        }
    
    for attempt in range(max_retries):
        try:
            response = requests.get(
                url, 
                headers=headers, 
                proxies=proxies,
                timeout=10
            )
            
            if response.status_code == 200:
                return response.text
            else:
                print(f"Attempt {attempt+1} failed with status code: {response.status_code}")
                
        except Exception as e:
            print(f"Attempt {attempt+1} failed with error: {str(e)}")
        
        # Wait before retrying
        if attempt < max_retries - 1:
            time.sleep(2 * (attempt + 1))  # Exponential backoff
    
    return None

# Example usage
proxy = "123.45.67.89:8080"  # Replace with your proxy
html = fetch("https://books.toscrape.com/", proxy)

if html:
    soup = BeautifulSoup(html, 'html.parser')
    books = soup.select('article.product_pod')
    
    for book in books[:5]:
        title = book.h3.a['title']
        print(f"Title: {title}")
else:
    print("Failed to fetch content after multiple retries")

import requests
from bs4 import BeautifulSoup
import json
import csv
import time
import random
from urllib3.exceptions import InsecureRequestWarning

# Disable SSL warnings
requests.packages.urllib3.disable_warnings(category=InsecureRequestWarning)

class AmazonScraper:
    def __init__(self, proxy_username, proxy_password, proxy_endpoint):
        self.session = requests.Session()
        
        # Configure proxy
        self.proxy = f"http://{proxy_username}:{proxy_password}@{proxy_endpoint}"
        self.proxies = {
            "http": self.proxy,
            "https": self.proxy
        }
        
        # Set headers to mimic a real browser
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.124 Safari/537.36",
            "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8",
            "Accept-Language": "en-US,en;q=0.5",
            "Accept-Encoding": "gzip, deflate, br",
            "Connection": "keep-alive",
            "Upgrade-Insecure-Requests": "1"
        }
    
    def scrape_amazon_books(self, search_term, num_pages=1):
        books = []
        base_url = "https://www.amazon.com/s?k="
        
        for page in range(1, num_pages + 1):
            url = f"{base_url}{search_term.replace(' ', '+')}&page={page}"
            print(f"Scraping page {page}: {url}")
            
            html = self.fetch_page(url)
            if not html:
                print(f"Failed to fetch page {page}")
                continue
                
            # Check if we hit a CAPTCHA
            if "Enter the characters you see below" in html or "To discuss automated access to Amazon data" in html:
                print("CAPTCHA detected! Proxy might be blocked.")
                continue
                
            page_books = self.parse_books(html)
            books.extend(page_books)
            
            # Be polite, don't hammer the server
            time.sleep(random.uniform(2, 5))
            
        return books
    
    def fetch_page(self, url, max_retries=3):
        for attempt in range(max_retries):
            try:
                response = self.session.get(
                    url,
                    headers=self.headers,
                    proxies=self.proxies,
                    timeout=20,
                    verify=False
                )
                
                if response.status_code == 200:
                    return response.text
                else:
                    print(f"Attempt {attempt+1} failed with status code: {response.status_code}")
                    
            except Exception as e:
                print(f"Attempt {attempt+1} failed with error: {str(e)}")
            
            # Wait before retrying
            if attempt < max_retries - 1:
                time.sleep(2 * (attempt + 1))
        
        return None
    
    def parse_books(self, html):
        soup = BeautifulSoup(html, 'html.parser')
        books = []
        
        # Find all book containers
        results = soup.select('div[data-component-type="s-search-result"]')
        
        for result in results:
            try:
                # Extract book title
                title_element = result.select_one('h2 a span')
                title = title_element.text.strip() if title_element else "Unknown Title"
                
                # Extract author
                author_element = result.select_one('a.a-size-base.a-link-normal.s-underline-text')
                author = author_element.text.strip() if author_element else "Unknown Author"
                
                # Extract price
                price_element = result.select_one('span.a-price > span.a-offscreen')
                price = price_element.text.strip() if price_element else "N/A"
                
                # Extract product URL
                url_element = result.select_one('h2 a')
                product_url = "https://www.amazon.com" + url_element['href'] if url_element and 'href' in url_element.attrs else ""
                
                books.append({
                    'title': title,
                    'author': author,
                    'price': price,
                    'url': product_url,
                    'timestamp': time.strftime("%Y-%m-%d %H:%M:%S")
                })
                
            except Exception as e:
                print(f"Error parsing book: {str(e)}")
        
        return books
    
    def save_to_csv(self, books, filename="amazon_books.csv"):
        if not books:
            print("No books to save")
            return
            
        with open(filename, 'w', newline='', encoding='utf-8') as f:
            writer = csv.DictWriter(f, fieldnames=books[0].keys())
            writer.writeheader()
            writer.writerows(books)
        
        print(f"Saved {len(books)} books to {filename}")
    
    def save_to_json(self, books, filename="amazon_books.json"):
        if not books:
            print("No books to save")
            return
            
        with open(filename, 'w', encoding='utf-8') as f:
            json.dump(books, f, ensure_ascii=False, indent=4)
        
        print(f"Saved {len(books)} books to {filename}")

def main():
    # Your proxy service credentials
    proxy_username = "your_username"
    proxy_password = "your_password"
    proxy_endpoint = "proxy.provider.com:12345"
    
    scraper = AmazonScraper(proxy_username, proxy_password, proxy_endpoint)
    books = scraper.scrape_amazon_books("python programming", num_pages=2)
    
    # Save the results
    scraper.save_to_csv(books)
    scraper.save_to_json(books)
    
    # Print sample results
    for book in books[:3]:
        print(f"Title: {book['title']}")
        print(f"Author: {book['author']}")
        print(f"Price: {book['price']}")
        print("---")

if __name__ == "__main__":
    main()