import type { Page } from "playwright"; import type { ExtractorMap, SpecEntry, JsonObject, JsonValue } from "../types.js"; async function getJsonLd(page: Page): Promise { return page.evaluate(() => { const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]')); return scripts.map(s => { try { const parsed = JSON.parse(s.textContent || "{}"); return typeof parsed === 'object' && parsed !== null ? parsed : {}; } catch { return {}; } }); }); } async function findJsonLdType(page: Page, typeStr: string): Promise { const jsonLds = await getJsonLd(page); for (const item of jsonLds) { const typeObj = item["@type"]; if (typeObj === typeStr || (Array.isArray(typeObj) && typeObj.includes(typeStr))) { return item; } const graphObj = item["@graph"]; if (graphObj && Array.isArray(graphObj)) { const found = graphObj.find((g: JsonValue) => g && typeof g === 'object' && !Array.isArray(g) && g["@type"] === typeStr); if (found) return found as JsonObject; } } return null; } export const extractors: ExtractorMap = { url: async (page) => { return page.url(); }, item_id: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.sku) return String(productData.sku); if (productData?.productID) return String(productData.productID); // Try finding data attributes or hidden inputs return page.evaluate(() => { const skuEl = document.querySelector('[data-product-sku], [data-sku], input[name="product_id"]'); if (skuEl) { return skuEl.getAttribute('data-product-sku') || skuEl.getAttribute('data-sku') || (skuEl as HTMLInputElement).value; } return null; }); }, title: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.name) return String(productData.name); return page.evaluate(() => { const imgAlt = document.querySelector('.product-detail-thumb-bto, .main-image img'); if (imgAlt && imgAlt.getAttribute('alt')) return imgAlt.getAttribute('alt')?.trim(); const titleStr = document.title; if (titleStr) { const dashPart = titleStr.split(' - ')[0] || titleStr; const pipePart = dashPart.split(' | ')[0] || dashPart; return pipePart.trim(); } const h1s = Array.from(document.querySelectorAll('h1')); const validH1 = h1s.find(el => { const text = el.textContent?.toLowerCase() || ''; return !text.includes('cookie') && !text.includes('choice') && !text.includes('consent'); }); if (validH1) return validH1.textContent?.trim(); return null; }); }, brand: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.brand && typeof productData.brand === 'object' && !Array.isArray(productData.brand)) { const brandName = (productData.brand as JsonObject).name; if (brandName) return String(brandName); } return page.evaluate(() => { const brandEl = document.querySelector('[itemprop="brand"]'); if (brandEl) return brandEl.textContent?.trim() || brandEl.getAttribute('content'); const siteName = document.querySelector('meta[property="og:site_name"]'); if (siteName) { const content = siteName.getAttribute('content'); if (content) return (content.split('-')[0] || content).trim(); } return null; }); }, product_category: async (page) => { return page.evaluate(() => { const breadcrumbs = Array.from(document.querySelectorAll('.breadcrumb li, .breadcrumbs li, [aria-label="breadcrumb"] li, [itemtype="http://schema.org/BreadcrumbList"] li, .path-item')); if (breadcrumbs.length > 0) { const texts = breadcrumbs.map(b => b.textContent?.replace(/\s+/g, ' ').trim()).filter(Boolean); const filtered = texts.filter(t => t && t.toLowerCase() !== 'home'); if (filtered.length > 0) return filtered.join(' > '); } return null; }); }, category_tree: async (page) => { return page.evaluate(() => { const items = Array.from(document.querySelectorAll('.breadcrumb a, .breadcrumbs a, [aria-label="breadcrumb"] a, [itemprop="itemListElement"] a')); if (items.length > 0) { const tree = items.map(el => { const name = el.textContent?.trim() || ''; const url = (el as HTMLAnchorElement).href || null; return { name, url }; }).filter(item => item.name && item.name.toLowerCase() !== 'home'); return tree; } return []; }); }, description: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.description) return String(productData.description); return page.evaluate(() => { const descEl = document.querySelector('.product-description, #description, [itemprop="description"]'); if (descEl) return descEl.textContent?.trim(); const metaDesc = document.querySelector('meta[name="description"], meta[property="og:description"]'); if (metaDesc) return metaDesc.getAttribute('content'); return null; }); }, price: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.offers) { if (Array.isArray(productData.offers)) { const offer = productData.offers[0] as JsonObject; if (offer?.price) return String(offer.price); } else if (typeof productData.offers === 'object') { const offer = productData.offers as JsonObject; if (offer.price) return String(offer.price); } } return page.evaluate(() => { const priceEl = document.querySelector('.prices-new, .price-new, [itemprop="price"], .product-price, .current-price, .regular-price'); if (priceEl) return priceEl.textContent?.trim(); return null; }); }, sale_price: async (page) => { return page.evaluate(() => { const hasOldPrice = document.querySelector('.prices-old, .price-old, .price-was'); if (hasOldPrice) { const saleEl = document.querySelector('.prices-new, .price-new'); if (saleEl) return saleEl.textContent?.trim(); } return null; }); }, availability: async (page) => { const productData = await findJsonLdType(page, "Product"); let offerUrl: unknown = null; if (productData?.offers) { if (Array.isArray(productData.offers)) { const offer = productData.offers[0] as JsonObject; offerUrl = offer?.availability; } else if (typeof productData.offers === 'object') { const offer = productData.offers as JsonObject; offerUrl = offer.availability; } } if (offerUrl) { return String(offerUrl).split('/').pop() || String(offerUrl); } return page.evaluate(() => { const btn = document.querySelector('#button-cart, .add-to-cart, [data-action="add-to-cart"], button.btn-cart'); if (btn) return btn.textContent?.trim() || "Add to Cart"; const stockEl = document.querySelector('.stock, .availability, [itemprop="availability"]'); if (stockEl) return stockEl.textContent?.trim() || stockEl.getAttribute('href')?.split('/').pop(); return null; }); }, image_url: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.image) { if (typeof productData.image === 'string') return productData.image; if (Array.isArray(productData.image)) return String(productData.image[0]); if (typeof productData.image === 'object') { const url = (productData.image as JsonObject).url; if (url) return String(url); } } return page.evaluate(() => { const img = document.querySelector('.product-detail-thumb-bto, .product-image img, .main-image img, [property="og:image"], img[itemprop="image"]'); if (img) { if (img.tagName.toLowerCase() === 'meta') return img.getAttribute('content'); return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src'); } return null; }); }, additional_image_urls: async (page) => { return page.evaluate(() => { const imgs = Array.from(document.querySelectorAll('.product-detail-thumb-bto, .gallery img, .thumbnails img, .product-thumbnails img')); return imgs.map(img => { return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src') || ''; }).filter(Boolean); }); }, specs: async (page) => { return page.evaluate(() => { const specs: SpecEntry[] = []; const rows = Array.from(document.querySelectorAll('.spec-table tr, .specifications tr, table tr, .tech-specs li')); for (const row of rows) { const th = row.querySelector('th'); const td = row.querySelector('td'); if (th && td) { specs.push({ name: th.textContent?.trim() || '', value: td.textContent?.trim() || null }); continue; } const label = row.querySelector('.label, .spec-name, dt'); const value = row.querySelector('.value, .spec-value, dd'); if (label && value) { specs.push({ name: label.textContent?.trim() || '', value: value.textContent?.trim() || null }); } } return specs.filter(s => s.name); }); }, star_rating: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) { const val = (productData.aggregateRating as JsonObject).ratingValue; if (val) return String(val); } return page.evaluate(() => { const ratingEl = document.querySelector('.rating-value, [itemprop="ratingValue"], .stars'); if (ratingEl) return ratingEl.textContent?.trim() || ratingEl.getAttribute('content'); return null; }); }, review_count: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) { const val = (productData.aggregateRating as JsonObject).reviewCount; if (val) return String(val); } return page.evaluate(() => { const countEl = document.querySelector('.review-count, [itemprop="reviewCount"]'); if (countEl) return countEl.textContent?.trim() || countEl.getAttribute('content'); return null; }); }, gtin: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.gtin13) return String(productData.gtin13); if (productData?.gtin) return String(productData.gtin); return page.evaluate(() => { const el = document.querySelector('[itemprop="gtin13"], [itemprop="gtin"]'); return el ? (el.textContent?.trim() || el.getAttribute('content')) : null; }); }, mpn: async (page) => { const productData = await findJsonLdType(page, "Product"); if (productData?.mpn) return String(productData.mpn); return page.evaluate(() => { const el = document.querySelector('[itemprop="mpn"]'); return el ? (el.textContent?.trim() || el.getAttribute('content')) : null; }); }, scraped_at: async () => { return new Date().toISOString(); } };