310 lines
11 KiB
TypeScript
310 lines
11 KiB
TypeScript
import type { Page } from "playwright";
|
|
import type { ExtractorMap, SpecEntry, JsonObject, JsonValue } from "../types.js";
|
|
|
|
async function getJsonLd(page: Page): Promise<JsonObject[]> {
|
|
return page.evaluate(() => {
|
|
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
|
return scripts.map(s => {
|
|
try {
|
|
const parsed = JSON.parse(s.textContent || "{}");
|
|
return typeof parsed === 'object' && parsed !== null ? parsed : {};
|
|
} catch {
|
|
return {};
|
|
}
|
|
});
|
|
});
|
|
}
|
|
|
|
async function findJsonLdType(page: Page, typeStr: string): Promise<JsonObject | null> {
|
|
const jsonLds = await getJsonLd(page);
|
|
for (const item of jsonLds) {
|
|
const typeObj = item["@type"];
|
|
if (typeObj === typeStr || (Array.isArray(typeObj) && typeObj.includes(typeStr))) {
|
|
return item;
|
|
}
|
|
const graphObj = item["@graph"];
|
|
if (graphObj && Array.isArray(graphObj)) {
|
|
const found = graphObj.find((g: JsonValue) => g && typeof g === 'object' && !Array.isArray(g) && g["@type"] === typeStr);
|
|
if (found) return found as JsonObject;
|
|
}
|
|
}
|
|
return null;
|
|
}
|
|
|
|
export const extractors: ExtractorMap = {
|
|
url: async (page) => {
|
|
return page.url();
|
|
},
|
|
|
|
item_id: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.sku) return String(productData.sku);
|
|
if (productData?.productID) return String(productData.productID);
|
|
|
|
// Try finding data attributes or hidden inputs
|
|
return page.evaluate(() => {
|
|
const skuEl = document.querySelector('[data-product-sku], [data-sku], input[name="product_id"]');
|
|
if (skuEl) {
|
|
return skuEl.getAttribute('data-product-sku') || skuEl.getAttribute('data-sku') || (skuEl as HTMLInputElement).value;
|
|
}
|
|
return null;
|
|
});
|
|
},
|
|
|
|
title: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.name) return String(productData.name);
|
|
|
|
return page.evaluate(() => {
|
|
const imgAlt = document.querySelector('.product-detail-thumb-bto, .main-image img');
|
|
if (imgAlt && imgAlt.getAttribute('alt')) return imgAlt.getAttribute('alt')?.trim();
|
|
|
|
const titleStr = document.title;
|
|
if (titleStr) {
|
|
const dashPart = titleStr.split(' - ')[0] || titleStr;
|
|
const pipePart = dashPart.split(' | ')[0] || dashPart;
|
|
return pipePart.trim();
|
|
}
|
|
|
|
const h1s = Array.from(document.querySelectorAll('h1'));
|
|
const validH1 = h1s.find(el => {
|
|
const text = el.textContent?.toLowerCase() || '';
|
|
return !text.includes('cookie') && !text.includes('choice') && !text.includes('consent');
|
|
});
|
|
if (validH1) return validH1.textContent?.trim();
|
|
|
|
return null;
|
|
});
|
|
},
|
|
|
|
brand: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.brand && typeof productData.brand === 'object' && !Array.isArray(productData.brand)) {
|
|
const brandName = (productData.brand as JsonObject).name;
|
|
if (brandName) return String(brandName);
|
|
}
|
|
|
|
return page.evaluate(() => {
|
|
const brandEl = document.querySelector('[itemprop="brand"]');
|
|
if (brandEl) return brandEl.textContent?.trim() || brandEl.getAttribute('content');
|
|
|
|
const siteName = document.querySelector('meta[property="og:site_name"]');
|
|
if (siteName) {
|
|
const content = siteName.getAttribute('content');
|
|
if (content) return (content.split('-')[0] || content).trim();
|
|
}
|
|
return null;
|
|
});
|
|
},
|
|
|
|
product_category: async (page) => {
|
|
return page.evaluate(() => {
|
|
const breadcrumbs = Array.from(document.querySelectorAll('.breadcrumb li, .breadcrumbs li, [aria-label="breadcrumb"] li, [itemtype="http://schema.org/BreadcrumbList"] li, .path-item'));
|
|
if (breadcrumbs.length > 0) {
|
|
const texts = breadcrumbs.map(b => b.textContent?.replace(/\s+/g, ' ').trim()).filter(Boolean);
|
|
const filtered = texts.filter(t => t && t.toLowerCase() !== 'home');
|
|
if (filtered.length > 0) return filtered.join(' > ');
|
|
}
|
|
return null;
|
|
});
|
|
},
|
|
|
|
category_tree: async (page) => {
|
|
return page.evaluate(() => {
|
|
const items = Array.from(document.querySelectorAll('.breadcrumb a, .breadcrumbs a, [aria-label="breadcrumb"] a, [itemprop="itemListElement"] a'));
|
|
if (items.length > 0) {
|
|
const tree = items.map(el => {
|
|
const name = el.textContent?.trim() || '';
|
|
const url = (el as HTMLAnchorElement).href || null;
|
|
return { name, url };
|
|
}).filter(item => item.name && item.name.toLowerCase() !== 'home');
|
|
return tree;
|
|
}
|
|
return [];
|
|
});
|
|
},
|
|
|
|
description: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.description) return String(productData.description);
|
|
|
|
return page.evaluate(() => {
|
|
const descEl = document.querySelector('.product-description, #description, [itemprop="description"]');
|
|
if (descEl) return descEl.textContent?.trim();
|
|
const metaDesc = document.querySelector('meta[name="description"], meta[property="og:description"]');
|
|
if (metaDesc) return metaDesc.getAttribute('content');
|
|
return null;
|
|
});
|
|
},
|
|
|
|
price: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.offers) {
|
|
if (Array.isArray(productData.offers)) {
|
|
const offer = productData.offers[0] as JsonObject;
|
|
if (offer?.price) return String(offer.price);
|
|
} else if (typeof productData.offers === 'object') {
|
|
const offer = productData.offers as JsonObject;
|
|
if (offer.price) return String(offer.price);
|
|
}
|
|
}
|
|
|
|
return page.evaluate(() => {
|
|
const priceEl = document.querySelector('.prices-new, .price-new, [itemprop="price"], .product-price, .current-price, .regular-price');
|
|
if (priceEl) return priceEl.textContent?.trim();
|
|
return null;
|
|
});
|
|
},
|
|
|
|
sale_price: async (page) => {
|
|
return page.evaluate(() => {
|
|
const hasOldPrice = document.querySelector('.prices-old, .price-old, .price-was');
|
|
if (hasOldPrice) {
|
|
const saleEl = document.querySelector('.prices-new, .price-new');
|
|
if (saleEl) return saleEl.textContent?.trim();
|
|
}
|
|
return null;
|
|
});
|
|
},
|
|
|
|
availability: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
let offerUrl: unknown = null;
|
|
|
|
if (productData?.offers) {
|
|
if (Array.isArray(productData.offers)) {
|
|
const offer = productData.offers[0] as JsonObject;
|
|
offerUrl = offer?.availability;
|
|
} else if (typeof productData.offers === 'object') {
|
|
const offer = productData.offers as JsonObject;
|
|
offerUrl = offer.availability;
|
|
}
|
|
}
|
|
|
|
if (offerUrl) {
|
|
return String(offerUrl).split('/').pop() || String(offerUrl);
|
|
}
|
|
|
|
return page.evaluate(() => {
|
|
const btn = document.querySelector('#button-cart, .add-to-cart, [data-action="add-to-cart"], button.btn-cart');
|
|
if (btn) return btn.textContent?.trim() || "Add to Cart";
|
|
|
|
const stockEl = document.querySelector('.stock, .availability, [itemprop="availability"]');
|
|
if (stockEl) return stockEl.textContent?.trim() || stockEl.getAttribute('href')?.split('/').pop();
|
|
return null;
|
|
});
|
|
},
|
|
|
|
image_url: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.image) {
|
|
if (typeof productData.image === 'string') return productData.image;
|
|
if (Array.isArray(productData.image)) return String(productData.image[0]);
|
|
if (typeof productData.image === 'object') {
|
|
const url = (productData.image as JsonObject).url;
|
|
if (url) return String(url);
|
|
}
|
|
}
|
|
|
|
return page.evaluate(() => {
|
|
const img = document.querySelector('.product-detail-thumb-bto, .product-image img, .main-image img, [property="og:image"], img[itemprop="image"]');
|
|
if (img) {
|
|
if (img.tagName.toLowerCase() === 'meta') return img.getAttribute('content');
|
|
return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src');
|
|
}
|
|
return null;
|
|
});
|
|
},
|
|
|
|
additional_image_urls: async (page) => {
|
|
return page.evaluate(() => {
|
|
const imgs = Array.from(document.querySelectorAll('.product-detail-thumb-bto, .gallery img, .thumbnails img, .product-thumbnails img'));
|
|
return imgs.map(img => {
|
|
return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src') || '';
|
|
}).filter(Boolean);
|
|
});
|
|
},
|
|
|
|
specs: async (page) => {
|
|
return page.evaluate(() => {
|
|
const specs: SpecEntry[] = [];
|
|
const rows = Array.from(document.querySelectorAll('.spec-table tr, .specifications tr, table tr, .tech-specs li'));
|
|
|
|
for (const row of rows) {
|
|
const th = row.querySelector('th');
|
|
const td = row.querySelector('td');
|
|
if (th && td) {
|
|
specs.push({
|
|
name: th.textContent?.trim() || '',
|
|
value: td.textContent?.trim() || null
|
|
});
|
|
continue;
|
|
}
|
|
|
|
const label = row.querySelector('.label, .spec-name, dt');
|
|
const value = row.querySelector('.value, .spec-value, dd');
|
|
if (label && value) {
|
|
specs.push({
|
|
name: label.textContent?.trim() || '',
|
|
value: value.textContent?.trim() || null
|
|
});
|
|
}
|
|
}
|
|
return specs.filter(s => s.name);
|
|
});
|
|
},
|
|
|
|
star_rating: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) {
|
|
const val = (productData.aggregateRating as JsonObject).ratingValue;
|
|
if (val) return String(val);
|
|
}
|
|
|
|
return page.evaluate(() => {
|
|
const ratingEl = document.querySelector('.rating-value, [itemprop="ratingValue"], .stars');
|
|
if (ratingEl) return ratingEl.textContent?.trim() || ratingEl.getAttribute('content');
|
|
return null;
|
|
});
|
|
},
|
|
|
|
review_count: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) {
|
|
const val = (productData.aggregateRating as JsonObject).reviewCount;
|
|
if (val) return String(val);
|
|
}
|
|
|
|
return page.evaluate(() => {
|
|
const countEl = document.querySelector('.review-count, [itemprop="reviewCount"]');
|
|
if (countEl) return countEl.textContent?.trim() || countEl.getAttribute('content');
|
|
return null;
|
|
});
|
|
},
|
|
|
|
gtin: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.gtin13) return String(productData.gtin13);
|
|
if (productData?.gtin) return String(productData.gtin);
|
|
|
|
return page.evaluate(() => {
|
|
const el = document.querySelector('[itemprop="gtin13"], [itemprop="gtin"]');
|
|
return el ? (el.textContent?.trim() || el.getAttribute('content')) : null;
|
|
});
|
|
},
|
|
|
|
mpn: async (page) => {
|
|
const productData = await findJsonLdType(page, "Product");
|
|
if (productData?.mpn) return String(productData.mpn);
|
|
|
|
return page.evaluate(() => {
|
|
const el = document.querySelector('[itemprop="mpn"]');
|
|
return el ? (el.textContent?.trim() || el.getAttribute('content')) : null;
|
|
});
|
|
},
|
|
|
|
scraped_at: async () => {
|
|
return new Date().toISOString();
|
|
}
|
|
};
|