feat: implement core product scraping infrastructure with Playwright extractors and data formatters

.gitignore

@@ -0,0 +1,9 @@
+node_modules/
+output/
+dist/
+build/
+.DS_Store
+.env
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "scraper",
+  "version": "1.0.0",
+  "description": "",
+  "main": "index.js",
+  "scripts": {
+    "scrape": "npx tsx src/index.ts"
+  },
+  "keywords": [],
+  "author": "",
+  "license": "ISC",
+  "type": "module",
+  "dependencies": {
+    "playwright": "^1.50.1",
+    "playwright-extra": "^4.3.6",
+    "puppeteer-extra-plugin-stealth": "^2.11.2"
+  },
+  "devDependencies": {
+    "@types/node": "^22.19.21",
+    "tsx": "^4.19.3",
+    "typescript": "^5.8.2"
+  }
+}

src/extract/extract-product.ts

@@ -0,0 +1,20 @@
+import type { Page } from "playwright";
+import type { RawProduct, Product } from "../types.js";
+import { extractors } from "./extractors.js";
+
+export const extractProduct = async (page: Page): Promise<RawProduct> => {
+  const raw: Partial<RawProduct> = {};
+
+  const keys = Object.keys(extractors) as Array<keyof Product>;
+
+  for (const key of keys) {
+    try {
+      raw[key] = await extractors[key](page);
+    } catch (error) {
+      console.error(`Error extracting field "${key}":`, error);
+      raw[key] = null;
+    }
+  }
+
+  return raw as RawProduct;
+};

src/extract/extractors.ts

@@ -0,0 +1,309 @@
+import type { Page } from "playwright";
+import type { ExtractorMap, SpecEntry, JsonObject, JsonValue } from "../types.js";
+
+async function getJsonLd(page: Page): Promise<JsonObject[]> {
+  return page.evaluate(() => {
+    const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
+    return scripts.map(s => {
+      try {
+        const parsed = JSON.parse(s.textContent || "{}");
+        return typeof parsed === 'object' && parsed !== null ? parsed : {};
+      } catch {
+        return {};
+      }
+    });
+  });
+}
+
+async function findJsonLdType(page: Page, typeStr: string): Promise<JsonObject | null> {
+  const jsonLds = await getJsonLd(page);
+  for (const item of jsonLds) {
+    const typeObj = item["@type"];
+    if (typeObj === typeStr || (Array.isArray(typeObj) && typeObj.includes(typeStr))) {
+      return item;
+    }
+    const graphObj = item["@graph"];
+    if (graphObj && Array.isArray(graphObj)) {
+      const found = graphObj.find((g: JsonValue) => g && typeof g === 'object' && !Array.isArray(g) && g["@type"] === typeStr);
+      if (found) return found as JsonObject;
+    }
+  }
+  return null;
+}
+
+export const extractors: ExtractorMap = {
+  url: async (page) => {
+    return page.url();
+  },
+
+  item_id: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.sku) return String(productData.sku);
+    if (productData?.productID) return String(productData.productID);
+
+    // Try finding data attributes or hidden inputs
+    return page.evaluate(() => {
+      const skuEl = document.querySelector('[data-product-sku], [data-sku], input[name="product_id"]');
+      if (skuEl) {
+        return skuEl.getAttribute('data-product-sku') || skuEl.getAttribute('data-sku') || (skuEl as HTMLInputElement).value;
+      }
+      return null;
+    });
+  },
+
+  title: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.name) return String(productData.name);
+
+    return page.evaluate(() => {
+      const imgAlt = document.querySelector('.product-detail-thumb-bto, .main-image img');
+      if (imgAlt && imgAlt.getAttribute('alt')) return imgAlt.getAttribute('alt')?.trim();
+
+      const titleStr = document.title;
+      if (titleStr) {
+        const dashPart = titleStr.split(' - ')[0] || titleStr;
+        const pipePart = dashPart.split(' | ')[0] || dashPart;
+        return pipePart.trim();
+      }
+
+      const h1s = Array.from(document.querySelectorAll('h1'));
+      const validH1 = h1s.find(el => {
+        const text = el.textContent?.toLowerCase() || '';
+        return !text.includes('cookie') && !text.includes('choice') && !text.includes('consent');
+      });
+      if (validH1) return validH1.textContent?.trim();
+
+      return null;
+    });
+  },
+
+  brand: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.brand && typeof productData.brand === 'object' && !Array.isArray(productData.brand)) {
+      const brandName = (productData.brand as JsonObject).name;
+      if (brandName) return String(brandName);
+    }
+
+    return page.evaluate(() => {
+      const brandEl = document.querySelector('[itemprop="brand"]');
+      if (brandEl) return brandEl.textContent?.trim() || brandEl.getAttribute('content');
+
+      const siteName = document.querySelector('meta[property="og:site_name"]');
+      if (siteName) {
+        const content = siteName.getAttribute('content');
+        if (content) return (content.split('-')[0] || content).trim();
+      }
+      return null;
+    });
+  },
+
+  product_category: async (page) => {
+    return page.evaluate(() => {
+      const breadcrumbs = Array.from(document.querySelectorAll('.breadcrumb li, .breadcrumbs li, [aria-label="breadcrumb"] li, [itemtype="http://schema.org/BreadcrumbList"] li, .path-item'));
+      if (breadcrumbs.length > 0) {
+        const texts = breadcrumbs.map(b => b.textContent?.replace(/\s+/g, ' ').trim()).filter(Boolean);
+        const filtered = texts.filter(t => t && t.toLowerCase() !== 'home');
+        if (filtered.length > 0) return filtered.join(' > ');
+      }
+      return null;
+    });
+  },
+
+  category_tree: async (page) => {
+    return page.evaluate(() => {
+      const items = Array.from(document.querySelectorAll('.breadcrumb a, .breadcrumbs a, [aria-label="breadcrumb"] a, [itemprop="itemListElement"] a'));
+      if (items.length > 0) {
+        const tree = items.map(el => {
+          const name = el.textContent?.trim() || '';
+          const url = (el as HTMLAnchorElement).href || null;
+          return { name, url };
+        }).filter(item => item.name && item.name.toLowerCase() !== 'home');
+        return tree;
+      }
+      return [];
+    });
+  },
+
+  description: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.description) return String(productData.description);
+
+    return page.evaluate(() => {
+      const descEl = document.querySelector('.product-description, #description, [itemprop="description"]');
+      if (descEl) return descEl.textContent?.trim();
+      const metaDesc = document.querySelector('meta[name="description"], meta[property="og:description"]');
+      if (metaDesc) return metaDesc.getAttribute('content');
+      return null;
+    });
+  },
+
+  price: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.offers) {
+      if (Array.isArray(productData.offers)) {
+        const offer = productData.offers[0] as JsonObject;
+        if (offer?.price) return String(offer.price);
+      } else if (typeof productData.offers === 'object') {
+        const offer = productData.offers as JsonObject;
+        if (offer.price) return String(offer.price);
+      }
+    }
+
+    return page.evaluate(() => {
+      const priceEl = document.querySelector('.prices-new, .price-new, [itemprop="price"], .product-price, .current-price, .regular-price');
+      if (priceEl) return priceEl.textContent?.trim();
+      return null;
+    });
+  },
+
+  sale_price: async (page) => {
+    return page.evaluate(() => {
+      const hasOldPrice = document.querySelector('.prices-old, .price-old, .price-was');
+      if (hasOldPrice) {
+        const saleEl = document.querySelector('.prices-new, .price-new');
+        if (saleEl) return saleEl.textContent?.trim();
+      }
+      return null;
+    });
+  },
+
+  availability: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    let offerUrl: unknown = null;
+
+    if (productData?.offers) {
+      if (Array.isArray(productData.offers)) {
+        const offer = productData.offers[0] as JsonObject;
+        offerUrl = offer?.availability;
+      } else if (typeof productData.offers === 'object') {
+        const offer = productData.offers as JsonObject;
+        offerUrl = offer.availability;
+      }
+    }
+
+    if (offerUrl) {
+      return String(offerUrl).split('/').pop() || String(offerUrl);
+    }
+
+    return page.evaluate(() => {
+      const btn = document.querySelector('#button-cart, .add-to-cart, [data-action="add-to-cart"], button.btn-cart');
+      if (btn) return btn.textContent?.trim() || "Add to Cart";
+
+      const stockEl = document.querySelector('.stock, .availability, [itemprop="availability"]');
+      if (stockEl) return stockEl.textContent?.trim() || stockEl.getAttribute('href')?.split('/').pop();
+      return null;
+    });
+  },
+
+  image_url: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.image) {
+      if (typeof productData.image === 'string') return productData.image;
+      if (Array.isArray(productData.image)) return String(productData.image[0]);
+      if (typeof productData.image === 'object') {
+        const url = (productData.image as JsonObject).url;
+        if (url) return String(url);
+      }
+    }
+
+    return page.evaluate(() => {
+      const img = document.querySelector('.product-detail-thumb-bto, .product-image img, .main-image img, [property="og:image"], img[itemprop="image"]');
+      if (img) {
+        if (img.tagName.toLowerCase() === 'meta') return img.getAttribute('content');
+        return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src');
+      }
+      return null;
+    });
+  },
+
+  additional_image_urls: async (page) => {
+    return page.evaluate(() => {
+      const imgs = Array.from(document.querySelectorAll('.product-detail-thumb-bto, .gallery img, .thumbnails img, .product-thumbnails img'));
+      return imgs.map(img => {
+        return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src') || '';
+      }).filter(Boolean);
+    });
+  },
+
+  specs: async (page) => {
+    return page.evaluate(() => {
+      const specs: SpecEntry[] = [];
+      const rows = Array.from(document.querySelectorAll('.spec-table tr, .specifications tr, table tr, .tech-specs li'));
+
+      for (const row of rows) {
+        const th = row.querySelector('th');
+        const td = row.querySelector('td');
+        if (th && td) {
+          specs.push({
+            name: th.textContent?.trim() || '',
+            value: td.textContent?.trim() || null
+          });
+          continue;
+        }
+
+        const label = row.querySelector('.label, .spec-name, dt');
+        const value = row.querySelector('.value, .spec-value, dd');
+        if (label && value) {
+          specs.push({
+            name: label.textContent?.trim() || '',
+            value: value.textContent?.trim() || null
+          });
+        }
+      }
+      return specs.filter(s => s.name);
+    });
+  },
+
+  star_rating: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) {
+      const val = (productData.aggregateRating as JsonObject).ratingValue;
+      if (val) return String(val);
+    }
+
+    return page.evaluate(() => {
+      const ratingEl = document.querySelector('.rating-value, [itemprop="ratingValue"], .stars');
+      if (ratingEl) return ratingEl.textContent?.trim() || ratingEl.getAttribute('content');
+      return null;
+    });
+  },
+
+  review_count: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) {
+      const val = (productData.aggregateRating as JsonObject).reviewCount;
+      if (val) return String(val);
+    }
+
+    return page.evaluate(() => {
+      const countEl = document.querySelector('.review-count, [itemprop="reviewCount"]');
+      if (countEl) return countEl.textContent?.trim() || countEl.getAttribute('content');
+      return null;
+    });
+  },
+
+  gtin: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.gtin13) return String(productData.gtin13);
+    if (productData?.gtin) return String(productData.gtin);
+
+    return page.evaluate(() => {
+      const el = document.querySelector('[itemprop="gtin13"], [itemprop="gtin"]');
+      return el ? (el.textContent?.trim() || el.getAttribute('content')) : null;
+    });
+  },
+
+  mpn: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.mpn) return String(productData.mpn);
+
+    return page.evaluate(() => {
+      const el = document.querySelector('[itemprop="mpn"]');
+      return el ? (el.textContent?.trim() || el.getAttribute('content')) : null;
+    });
+  },
+
+  scraped_at: async () => {
+    return new Date().toISOString();
+  }
+};

src/format/format-product.ts

@@ -0,0 +1,14 @@
+import type { RawProduct, Product } from "../types.js";
+import { formatters } from "./formatters.js";
+
+export const formatProduct = (raw: RawProduct): Product => {
+  const product: Partial<Product> = {};
+
+  const keys = Object.keys(formatters) as Array<keyof Product>;
+
+  for (const key of keys) {
+    (product as Record<string, unknown>)[key] = formatters[key](raw[key]);
+  }
+
+  return product as Product;
+};

src/format/formatters.ts

@@ -0,0 +1,83 @@
+import type { FormatterMap, Availability, CategoryEntry, SpecEntry } from "../types.js";
+
+const availabilityMap: Record<string, Availability> = {
+  "add to cart": "in_stock",
+  "in stock": "in_stock",
+  "instock": "in_stock",
+  "out of stock": "out_of_stock",
+  "outofstock": "out_of_stock",
+  "sold out": "out_of_stock",
+  "pre-order": "pre_order",
+  "pre order": "pre_order",
+  "preorder": "pre_order"
+};
+
+const parseNumber = (val: unknown): number | null => {
+  if (typeof val === "number") return val;
+  if (typeof val !== "string") return null;
+  const cleaned = val.replace(/[^\d.-]/g, "");
+  const parsed = parseFloat(cleaned);
+  return isNaN(parsed) ? null : parsed;
+};
+
+const parseString = (val: unknown): string | null => {
+  if (typeof val === "string") {
+    const trimmed = val.trim();
+    return trimmed.length > 0 ? trimmed : null;
+  }
+  return null;
+};
+
+export const formatters: FormatterMap = {
+  url: (val) => parseString(val) || "",
+
+  item_id: parseString,
+
+  title: parseString,
+
+  brand: parseString,
+
+  product_category: parseString,
+
+  category_tree: (val) => {
+    if (!Array.isArray(val)) return [];
+    return val as CategoryEntry[];
+  },
+
+  description: parseString,
+
+  price: parseNumber,
+
+  sale_price: parseNumber,
+
+  availability: (val) => {
+    const str = parseString(val);
+    if (!str) return null;
+    const lower = str.toLowerCase();
+    return availabilityMap[lower] ?? null;
+  },
+
+  image_url: parseString,
+
+  additional_image_urls: (val) => {
+    if (!Array.isArray(val)) return [];
+    // Deduplicate array
+    const urls = val.map(parseString).filter(Boolean) as string[];
+    return Array.from(new Set(urls));
+  },
+
+  specs: (val) => {
+    if (!Array.isArray(val)) return [];
+    return val as SpecEntry[];
+  },
+
+  star_rating: parseNumber,
+
+  review_count: parseNumber,
+
+  gtin: parseString,
+
+  mpn: parseString,
+
+  scraped_at: (val) => parseString(val) || new Date().toISOString()
+};

src/index.ts

@@ -0,0 +1,45 @@
+import { writeFileSync, mkdirSync } from "node:fs";
+import { dirname } from "node:path";
+import { loadPage } from "./request/page-loader.js";
+import { extractProduct } from "./extract/extract-product.js";
+import { formatProduct } from "./format/format-product.js";
+
+const DEFAULT_URL = "https://us-store.msi.com/Motherboards/Intel-Platform-Motherboard/INTEL-Z890/MAG-Z890-TOMAHAWK-WIFI";
+const TARGET_URL = process.argv[2] || DEFAULT_URL;
+const OUTPUT_FILE = "output/product.json";
+
+async function main() {
+  console.log(`Starting scraper for URL: ${TARGET_URL}`);
+
+  let browser;
+  try {
+    const loaderResult = await loadPage(TARGET_URL);
+    browser = loaderResult.browser;
+    const page = loaderResult.page;
+
+    console.log("Page loaded. Extracting data...");
+    const rawData = await extractProduct(page);
+
+    console.log("Data extracted. Formatting...");
+    const product = formatProduct(rawData);
+
+    console.log("Formatting complete. Saving output...");
+    mkdirSync(dirname(OUTPUT_FILE), { recursive: true });
+    writeFileSync(OUTPUT_FILE, JSON.stringify(product, null, 2), "utf8");
+
+    const html = await page.content();
+    writeFileSync("output/page.html", html, "utf8");
+
+    console.log(`Success! Data saved to ${OUTPUT_FILE}`);
+  } catch (error) {
+    console.error("Scraping failed:", error);
+    process.exit(1);
+  } finally {
+    if (browser) {
+      await browser.close();
+      console.log("Browser closed.");
+    }
+  }
+}
+
+main();

src/request/page-loader.ts

@@ -0,0 +1,26 @@
+import { chromium } from "playwright-extra";
+import stealth from "puppeteer-extra-plugin-stealth";
+import type { Browser, Page } from "playwright";
+
+chromium.use(stealth());
+
+export const loadPage = async (url: string): Promise<{ page: Page; browser: Browser }> => {
+  const browser = await chromium.launch({ headless: true });
+  const context = await browser.newContext({
+    userAgent: "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36",
+    viewport: { width: 1920, height: 1080 }
+  });
+
+  const page = await context.newPage();
+
+  await page.goto(url, { waitUntil: "domcontentloaded" });
+  await page.waitForTimeout(3000);
+
+  try {
+    await page.waitForSelector("h1", { timeout: 10000 });
+  } catch (error) {
+    console.warn("Warning: Could not find <h1> element within timeout.");
+  }
+
+  return { page, browser };
+};

src/types.ts

@@ -0,0 +1,51 @@
+import type { Page } from "playwright";
+
+export type JsonPrimitive = string | number | boolean | null;
+export type JsonValue = JsonPrimitive | JsonObject | JsonArray;
+export interface JsonObject {
+  [key: string]: JsonValue;
+}
+export interface JsonArray extends Array<JsonValue> {}
+
+export type Availability = "in_stock" | "out_of_stock" | "pre_order";
+
+export type CategoryEntry = {
+  name: string;
+  url: string | null;
+};
+
+export type SpecEntry = {
+  name: string;
+  value: string | null;
+};
+
+export type Product = {
+  url: string;
+  item_id: string | null;
+  title: string | null;
+  brand: string | null;
+  product_category: string | null;
+  category_tree: CategoryEntry[];
+  description: string | null;
+  price: number | null;
+  sale_price: number | null;
+  availability: Availability | null;
+  image_url: string | null;
+  additional_image_urls: string[];
+  specs: SpecEntry[];
+  star_rating: number | null;
+  review_count: number | null;
+  gtin: string | null;
+  mpn: string | null;
+  scraped_at: string;
+};
+
+export type RawProduct = {
+  [K in keyof Product]: unknown;
+};
+
+export type Extractor = (page: Page) => Promise<unknown>;
+export type Formatter = (raw: unknown) => unknown;
+
+export type ExtractorMap = Record<keyof Product, Extractor>;
+export type FormatterMap = Record<keyof Product, Formatter>;

tsconfig.json

@@ -0,0 +1,44 @@
+{
+  // Visit https://aka.ms/tsconfig to read more about this file
+  "compilerOptions": {
+    // File Layout
+    // "rootDir": "./src",
+    // "outDir": "./dist",
+
+    // Environment Settings
+    // See also https://aka.ms/tsconfig/module
+    "module": "nodenext",
+    "target": "esnext",
+    "types": ["node"],
+    // For nodejs:
+    // "lib": ["esnext"],
+    // "types": ["node"],
+    // and npm install -D @types/node
+
+    // Other Outputs
+    "sourceMap": true,
+    "declaration": true,
+    "declarationMap": true,
+
+    // Stricter Typechecking Options
+    "noUncheckedIndexedAccess": true,
+    "exactOptionalPropertyTypes": true,
+
+    // Style Options
+    // "noImplicitReturns": true,
+    // "noImplicitOverride": true,
+    // "noUnusedLocals": true,
+    // "noUnusedParameters": true,
+    // "noFallthroughCasesInSwitch": true,
+    // "noPropertyAccessFromIndexSignature": true,
+
+    // Recommended Options
+    "strict": true,
+    "jsx": "react-jsx",
+    "verbatimModuleSyntax": true,
+    "isolatedModules": true,
+    "noUncheckedSideEffectImports": true,
+    "moduleDetection": "force",
+    "skipLibCheck": true,
+  }
+}