From Static HTML to Dynamic Pages: Two Practical Scrapers

      mkdir quotes-scraper && cd quotes-scraper
npm init -y
    

      npm i axios cheerio
    

      import axios from "axios";
import * as cheerio from "cheerio";

const BASE = "https://quotes.toscrape.com/";

async function scrapeQuotes(page = 1) {
  const url = page === 1 ? BASE : `${BASE}page/${page}/`;
  const { data: html } = await axios.get(url, {
    headers: {
      "User-Agent":
        "Mozilla/5.0 (compatible; QuotesScraper/1.0; +https://example.com/bot)"
    },
    timeout: 15000
  });

  const $ = cheerio.load(html);
  const items = [];

  $(".quote").each((_, el) => {
    const text = $(el).find(".text").text().replace(/^“|”$/g, "").trim();
    const author = $(el).find(".author").text().trim();
    const tags = $(el)
      .find(".tags a.tag")
      .map((_, a) => $(a).text().trim())
      .get();

    items.push({ text, author, tags });
  });

  const nextHref = $(".pager .next a").attr("href") || null;
  const hasNext = Boolean(nextHref);

  return { items, hasNext };
}

(async () => {
  try {
    const all = [];
    let page = 1;

    while (true) {
      const { items, hasNext } = await scrapeQuotes(page);
      all.push(...items);
      console.log(`[cheerio] page ${page}: +${items.length} quotes`);
      if (!hasNext) break;
      page += 1;
    }

    // Show first 2 results to keep console readable
    console.log(JSON.stringify(all.slice(0, 2), null, 2));
    console.log(`Total quotes collected: ${all.length}`);
  } catch (err) {
    console.error("Cheerio scrape failed:", err.message);
    process.exit(1);
  }
})();
    

      [
  {
    "text": "The world as we have created it is a process of our thinking. It cannot be changed without changing our thinking.",
    "author": "Albert Einstein",
    "tags": ["change", "deep-thoughts", "thinking", "world"]
  },
  {
    "text": "It is our choices, Harry, that show what we truly are, far more than our abilities.",
    "author": "J.K. Rowling",
    "tags": ["abilities", "choices"]
  }
]
    

      node quotes-cheerio.mjs
    

      npm i puppeteer
    

      import puppeteer from "puppeteer";

const BASE = "https://quotes.toscrape.com/";

async function scrapeQuotesWithPuppeteer() {
  const browser = await puppeteer.launch({
    headless: "new", // Chrome headless mode
    args: ["--no-sandbox", "--disable-setuid-sandbox"]
  });

  try {
    const page = await browser.newPage();
    await page.setUserAgent(
      "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " +
        "AppleWebKit/537.36 (KHTML, like Gecko) " +
        "Chrome/120.0.0.0 Safari/537.36"
    );

    const all = [];
    let url = BASE;

    while (true) {
      await page.goto(url, { waitUntil: "domcontentloaded", timeout: 30000 });

      // Optionally wait for quotes to be present
      await page.waitForSelector(".quote", { timeout: 10000 });

      // Extract structured data in the browser context
      const items = await page.$$eval(".quote", (cards) =>
        cards.map((el) => {
          const text = el.querySelector(".text")?.textContent || "";
          const author = el.querySelector(".author")?.textContent || "";
          const tags = Array.from(el.querySelectorAll(".tags a.tag")).map((a) =>
            a.textContent.trim()
          );
          return {
            text: text.replace(/^“|”$/g, "").trim(),
            author: author.trim(),
            tags
          };
        })
      );

      all.push(...items);

      const nextHref = await page.$eval(
        ".pager",
        (pager) => pager.querySelector(".next a")?.getAttribute("href") || null
      ).catch(() => null);

      console.log(`[puppeteer] page ${url}: +${items.length} quotes`);

      if (!nextHref) break;
      url = new URL(nextHref, location.origin).href;
    }

    console.log(JSON.stringify(all.slice(0, 2), null, 2));
    console.log(`Total quotes collected: ${all.length}`);
  } catch (err) {
    console.error("Puppeteer scrape failed:", err.message);
  } finally {
    await browser.close();
  }
}

scrapeQuotesWithPuppeteer();
    

      node quotes-puppeteer.mjs
    

      import fs from "fs";

fs.writeFileSync("quotes.json", JSON.stringify(all, null, 2), "utf-8");
console.log("Saved to quotes.json");