from crawlbase import CrawlingAPI

# Initialize Crawlbase API with your access token
crawling_api = CrawlingAPI({ 'token': 'YOUR_CRAWLBASE_TOKEN' })

def make_crawlbase_request(url):
    response = crawling_api.get(url)

    if response['headers']['pc_status'] == '200':
        html_content = response['body'].decode('utf-8')
        return html_content
    else:
        print(f"Failed to fetch the page. Crawlbase status code: {response['headers']['pc_status']}")
        return None

python --version

pip install crawlbase
pip install beautifulsoup4

https://www.goodreads.com/book/show/4671.The_Great_Gatsby/reviews

from crawlbase import CrawlingAPI
import json
from bs4 import BeautifulSoup

# Initialize Crawlbase API with JS Token
crawling_api = CrawlingAPI({ 'token': 'CRAWLBASE_JS_TOKEN' })

# Function to fetch and process Goodreads book details and reviews
def scrape_goodreads_reviews(base_url):
    page_data = []

    # Fetch initial page and reviews
    response = crawling_api.get(base_url, {
        'ajax_wait': 'true',
        'page_wait': '5000'
    })

    if response['headers']['pc_status'] == '200':
        html_content = response['body'].decode('utf-8')
        page_data = extract_book_details(html_content)

    return page_data

# Function to extract the book title, rating, and reviews from the page
def extract_book_details(html):
    soup = BeautifulSoup(html, 'html.parser')
    title = soup.select_one('h1.H1Title a[data-testid="title"]').text.strip()
    rating = soup.select_one('div.RatingStatistics span.RatingStars')['aria-label']

    reviews = []
    for review_div in soup.select('div.ReviewsList article.ReviewCard'):
        user = review_div.select_one('div[data-testid="name"]').text.strip()
        review_text = review_div.select_one('section.ReviewText span.Formatted').text.strip()
        reviews.append({'user': user, 'review': review_text})

    return {'title': title, 'rating': rating, 'reviews': reviews}

def scrape_goodreads_reviews_with_pagination(base_url):
    page_data = []

    # Fetch initial page and reviews
    response = crawling_api.get(base_url, {
        'ajax_wait': 'true',
        'page_wait': '5000',
        'css_click_selector': 'button:has(span[data-testid="loadMore"])'
    })

    if response['headers']['pc_status'] == '200':
        html_content = response['body'].decode('utf-8')
        page_data = extract_book_details(html_content)

    return page_data

# Function to save scraped reviews to a JSON file
def save_reviews_to_json(data, filename='goodreads_reviews.json'):
    with open(filename, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=4)

# Example usage
book_reviews = scrape_goodreads_reviews_with_pagination('https://www.goodreads.com/book/show/4671.The_Great_Gatsby/reviews')
save_reviews_to_json(book_reviews)

from crawlbase import CrawlingAPI
import json
from bs4 import BeautifulSoup

# Initialize Crawlbase API with JS Token
crawling_api = CrawlingAPI({ 'token': 'CRAWLBASE_JS_TOKEN' })

# Function to extract book details and reviews from the HTML content
def extract_book_details(html):
    soup = BeautifulSoup(html, 'html.parser')
    title = soup.select_one('h1.H1Title a[data-testid="title"]').text.strip()
    rating = soup.select_one('div.RatingStatistics span.RatingStars')['aria-label']

    reviews = []
    for review_div in soup.select('div.ReviewsList article.ReviewCard'):
        user = review_div.select_one('div[data-testid="name"]').text.strip()
        review_text = review_div.select_one('section.ReviewText span.Formatted').text.strip()
        reviews.append({'user': user, 'review': review_text})

    return {'title': title, 'rating': rating, 'reviews': reviews}

# Function to scrape Goodreads with pagination
def scrape_goodreads_reviews_with_pagination(base_url):
    page_data = []

    # Fetch initial page and reviews
    response = crawling_api.get(base_url, {
        'ajax_wait': 'true',
        'page_wait': '5000',
        'css_click_selector': 'button:has(span[data-testid="loadMore"])'
    })

    if response['headers']['pc_status'] == '200':
        html_content = response['body'].decode('utf-8')
        page_data = extract_book_details(html_content)

    return page_data

# Function to save the reviews in JSON format
def save_reviews_to_json(data, filename='goodreads_reviews.json'):
    with open(filename, 'w', encoding='utf-8') as f:
        json.dump(data, f, ensure_ascii=False, indent=4)

# Example usage
book_reviews = scrape_goodreads_reviews_with_pagination('https://www.goodreads.com/book/show/4671.The_Great_Gatsby/reviews')
save_reviews_to_json(book_reviews)

{
    "title": "The Great Gatsby",
    "rating": "Rating 3.93 out of 5",
    "reviews": [
        {
            "user": "Alex",
            "review": "The Great Gatsby is your neighbor you're best friends with until you find out he's a drug dealer. It charms you with some of the most elegant English prose ever published, making it difficult to discuss the novel without the urge to stammer awestruck about its beauty. It would be evidence enough to argue that F. Scott Fitzgerald was superhuman, if it wasn't for the fact that we know he also wrote This Side of Paradise.But despite its magic, the rhetoric is just that, and it is a cruel facade. Behind the stunning glitter lies a story with all the discontent and intensity of the early Metallica albums. At its heart, The Great Gatsby throws the very nature of our desires into a harsh, shocking light. There may never be a character who so epitomizes tragically misplaced devotion as Jay Gatsby, and Daisy, his devotee, plays her part with perfect, innocent malevolence. Gatsby's competition, Tom Buchanan, stands aside watching, taunting and provoking with piercing vocal jabs and the constant boast of his enviable physique. The three jostle for position in an epic love triangle that lays waste to countless innocent victims, as well as both Eggs of Long Island. Every jab, hook, and uppercut is relayed by the instantly likable narrator Nick Carraway, seemingly the only voice of reason amongst all the chaos. But when those boats are finally borne back ceaselessly by the current, no one is left afloat. It is an ethical massacre, and Fitzgerald spares no lives; there is perhaps not a single character of any significance worthy even of a Sportsmanship Award from the Boys and Girls Club.In a word, The Great Gatsby is about deception; Fitzgerald tints our glasses rosy with gorgeous prose and a narrator you want so much to trust, but leaves the lenses just translucent enough for us to see that Gatsby is getting the same treatment. And if Gatsby represents the truth of the American Dream, it means trouble for us all. Consider it the most pleasant insult you'll ever receive."
        },
        {
            "user": "Lisa of Troy",
            "review": "Fitzgerald, you have ruined me.Fitzgerald can set a scene so perfectly, flawlessly. He paints a world of magic and introduces one of the greatest characters of all time, Jay Gatsby. Gatsby is the embodiment of hope, and no one can dissuade him from his dreams. Have you ever had a dream that carried you to heights you could never have dreamed otherwise? When Gatsby is reunited with Daisy Buchanan, he fills the space to the brim with flowers, creating a living dream. How is anyone supposed to compete with that?The Great Gatsby perfectly makes use of a narrator, Nick. Why is Gatsby so great? Because Nick tells us. If Gatsby told us, we would just think that he is a braggard, the least humble person in the world. This book is wildly addictive, so intricate yet perfectly woven together, a brilliant literary masterpiece. I have to keep going back to reconnect with Jay Gatsby, a naïve but beautiful and charming hope, perfectly imperfect, a relentless dreamer.2025 Reading ScheduleJan\tA Town Like AliceFeb\tBirdsongMar\tCaptain Corelli's Mandolin - Louis De BerniereApr\tWar and PeaceMay\tThe Woman in WhiteJun\tAtonementJul\tThe Shadow of the WindAug\tJude the ObscureSep\tUlyssesOct\tVanity FairNov\tA Fine BalanceDec\tGerminalConnect With Me!Blog Twitter BookTube Facebook Insta My Bookstore at Pango"
        },
        {
            "user": "Kemper",
            "review": "Jay Gatsby, you poor doomed bastard. You were ahead of your time. If you would have pulled your scam after the invention of reality TV, you would have been a huge star on a show like The Bachelor and a dozen shameless Daisy-types would have thrown themselves at you. Mass media and modern fame would have embraced the way you tried to push your way into a social circle you didn’t belong to in an effort to fulfill a fool’s dream as your entire existence became a lie and you desperately sought to rewrite history to an ending you wanted. You had a talent for it, Jay, but a modern PR expert would have made you bigger than Kate Gosselin. Your knack for self-promotion and over the top displays of wealth to try and buy respectability would have fit right in these days. I can just about see you on a red carpet with Paris Hilton. And the ending would have been different. No aftermath for rich folks these days. Lawyers and pay-off money would have quietly settled the matter. No harm, no foul. But then you’d have realized how worthless Daisy really was at some point. I’m sure you couldn’t have dealt with that. So maybe it is better that your story happened in the Jazz Age where you could keep your illusions intact to the bitter end.The greatest American novel? I don’t know if there is such an animal. But I think you'd have to include this one in the conversation."
        },
        {
            "user": "Inge",
            "review": "There was one thing I really liked about The Great Gatsby.It was short."
        },
        {
            "user": "may ➹",
            "review": "the only thing I got from this is that Nick is gay2.5"
        },
        .... more
    ]
}