feat: implement core product scraping infrastructure with Playwright extractors and data formatters

2026-06-19 14:15:50 +03:00
commit 59b5c73612
11 changed files with 1753 additions and 0 deletions
@@ -0,0 +1,309 @@
+import type { Page } from "playwright";
+import type { ExtractorMap, SpecEntry, JsonObject, JsonValue } from "../types.js";
+
+async function getJsonLd(page: Page): Promise<JsonObject[]> {
+  return page.evaluate(() => {
+    const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
+    return scripts.map(s => {
+      try {
+        const parsed = JSON.parse(s.textContent || "{}");
+        return typeof parsed === 'object' && parsed !== null ? parsed : {};
+      } catch {
+        return {};
+      }
+    });
+  });
+}
+
+async function findJsonLdType(page: Page, typeStr: string): Promise<JsonObject | null> {
+  const jsonLds = await getJsonLd(page);
+  for (const item of jsonLds) {
+    const typeObj = item["@type"];
+    if (typeObj === typeStr || (Array.isArray(typeObj) && typeObj.includes(typeStr))) {
+      return item;
+    }
+    const graphObj = item["@graph"];
+    if (graphObj && Array.isArray(graphObj)) {
+      const found = graphObj.find((g: JsonValue) => g && typeof g === 'object' && !Array.isArray(g) && g["@type"] === typeStr);
+      if (found) return found as JsonObject;
+    }
+  }
+  return null;
+}
+
+export const extractors: ExtractorMap = {
+  url: async (page) => {
+    return page.url();
+  },
+
+  item_id: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.sku) return String(productData.sku);
+    if (productData?.productID) return String(productData.productID);
+
+    // Try finding data attributes or hidden inputs
+    return page.evaluate(() => {
+      const skuEl = document.querySelector('[data-product-sku], [data-sku], input[name="product_id"]');
+      if (skuEl) {
+        return skuEl.getAttribute('data-product-sku') || skuEl.getAttribute('data-sku') || (skuEl as HTMLInputElement).value;
+      }
+      return null;
+    });
+  },
+
+  title: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.name) return String(productData.name);
+
+    return page.evaluate(() => {
+      const imgAlt = document.querySelector('.product-detail-thumb-bto, .main-image img');
+      if (imgAlt && imgAlt.getAttribute('alt')) return imgAlt.getAttribute('alt')?.trim();
+
+      const titleStr = document.title;
+      if (titleStr) {
+        const dashPart = titleStr.split(' - ')[0] || titleStr;
+        const pipePart = dashPart.split(' | ')[0] || dashPart;
+        return pipePart.trim();
+      }
+
+      const h1s = Array.from(document.querySelectorAll('h1'));
+      const validH1 = h1s.find(el => {
+        const text = el.textContent?.toLowerCase() || '';
+        return !text.includes('cookie') && !text.includes('choice') && !text.includes('consent');
+      });
+      if (validH1) return validH1.textContent?.trim();
+
+      return null;
+    });
+  },
+
+  brand: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.brand && typeof productData.brand === 'object' && !Array.isArray(productData.brand)) {
+      const brandName = (productData.brand as JsonObject).name;
+      if (brandName) return String(brandName);
+    }
+
+    return page.evaluate(() => {
+      const brandEl = document.querySelector('[itemprop="brand"]');
+      if (brandEl) return brandEl.textContent?.trim() || brandEl.getAttribute('content');
+
+      const siteName = document.querySelector('meta[property="og:site_name"]');
+      if (siteName) {
+        const content = siteName.getAttribute('content');
+        if (content) return (content.split('-')[0] || content).trim();
+      }
+      return null;
+    });
+  },
+
+  product_category: async (page) => {
+    return page.evaluate(() => {
+      const breadcrumbs = Array.from(document.querySelectorAll('.breadcrumb li, .breadcrumbs li, [aria-label="breadcrumb"] li, [itemtype="http://schema.org/BreadcrumbList"] li, .path-item'));
+      if (breadcrumbs.length > 0) {
+        const texts = breadcrumbs.map(b => b.textContent?.replace(/\s+/g, ' ').trim()).filter(Boolean);
+        const filtered = texts.filter(t => t && t.toLowerCase() !== 'home');
+        if (filtered.length > 0) return filtered.join(' > ');
+      }
+      return null;
+    });
+  },
+
+  category_tree: async (page) => {
+    return page.evaluate(() => {
+      const items = Array.from(document.querySelectorAll('.breadcrumb a, .breadcrumbs a, [aria-label="breadcrumb"] a, [itemprop="itemListElement"] a'));
+      if (items.length > 0) {
+        const tree = items.map(el => {
+          const name = el.textContent?.trim() || '';
+          const url = (el as HTMLAnchorElement).href || null;
+          return { name, url };
+        }).filter(item => item.name && item.name.toLowerCase() !== 'home');
+        return tree;
+      }
+      return [];
+    });
+  },
+
+  description: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.description) return String(productData.description);
+
+    return page.evaluate(() => {
+      const descEl = document.querySelector('.product-description, #description, [itemprop="description"]');
+      if (descEl) return descEl.textContent?.trim();
+      const metaDesc = document.querySelector('meta[name="description"], meta[property="og:description"]');
+      if (metaDesc) return metaDesc.getAttribute('content');
+      return null;
+    });
+  },
+
+  price: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.offers) {
+      if (Array.isArray(productData.offers)) {
+        const offer = productData.offers[0] as JsonObject;
+        if (offer?.price) return String(offer.price);
+      } else if (typeof productData.offers === 'object') {
+        const offer = productData.offers as JsonObject;
+        if (offer.price) return String(offer.price);
+      }
+    }
+
+    return page.evaluate(() => {
+      const priceEl = document.querySelector('.prices-new, .price-new, [itemprop="price"], .product-price, .current-price, .regular-price');
+      if (priceEl) return priceEl.textContent?.trim();
+      return null;
+    });
+  },
+
+  sale_price: async (page) => {
+    return page.evaluate(() => {
+      const hasOldPrice = document.querySelector('.prices-old, .price-old, .price-was');
+      if (hasOldPrice) {
+        const saleEl = document.querySelector('.prices-new, .price-new');
+        if (saleEl) return saleEl.textContent?.trim();
+      }
+      return null;
+    });
+  },
+
+  availability: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    let offerUrl: unknown = null;
+
+    if (productData?.offers) {
+      if (Array.isArray(productData.offers)) {
+        const offer = productData.offers[0] as JsonObject;
+        offerUrl = offer?.availability;
+      } else if (typeof productData.offers === 'object') {
+        const offer = productData.offers as JsonObject;
+        offerUrl = offer.availability;
+      }
+    }
+
+    if (offerUrl) {
+      return String(offerUrl).split('/').pop() || String(offerUrl);
+    }
+
+    return page.evaluate(() => {
+      const btn = document.querySelector('#button-cart, .add-to-cart, [data-action="add-to-cart"], button.btn-cart');
+      if (btn) return btn.textContent?.trim() || "Add to Cart";
+
+      const stockEl = document.querySelector('.stock, .availability, [itemprop="availability"]');
+      if (stockEl) return stockEl.textContent?.trim() || stockEl.getAttribute('href')?.split('/').pop();
+      return null;
+    });
+  },
+
+  image_url: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.image) {
+      if (typeof productData.image === 'string') return productData.image;
+      if (Array.isArray(productData.image)) return String(productData.image[0]);
+      if (typeof productData.image === 'object') {
+        const url = (productData.image as JsonObject).url;
+        if (url) return String(url);
+      }
+    }
+
+    return page.evaluate(() => {
+      const img = document.querySelector('.product-detail-thumb-bto, .product-image img, .main-image img, [property="og:image"], img[itemprop="image"]');
+      if (img) {
+        if (img.tagName.toLowerCase() === 'meta') return img.getAttribute('content');
+        return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src');
+      }
+      return null;
+    });
+  },
+
+  additional_image_urls: async (page) => {
+    return page.evaluate(() => {
+      const imgs = Array.from(document.querySelectorAll('.product-detail-thumb-bto, .gallery img, .thumbnails img, .product-thumbnails img'));
+      return imgs.map(img => {
+        return img.getAttribute('popup_img') || (img as HTMLImageElement).src || img.getAttribute('data-src') || img.getAttribute('src') || '';
+      }).filter(Boolean);
+    });
+  },
+
+  specs: async (page) => {
+    return page.evaluate(() => {
+      const specs: SpecEntry[] = [];
+      const rows = Array.from(document.querySelectorAll('.spec-table tr, .specifications tr, table tr, .tech-specs li'));
+
+      for (const row of rows) {
+        const th = row.querySelector('th');
+        const td = row.querySelector('td');
+        if (th && td) {
+          specs.push({
+            name: th.textContent?.trim() || '',
+            value: td.textContent?.trim() || null
+          });
+          continue;
+        }
+
+        const label = row.querySelector('.label, .spec-name, dt');
+        const value = row.querySelector('.value, .spec-value, dd');
+        if (label && value) {
+          specs.push({
+            name: label.textContent?.trim() || '',
+            value: value.textContent?.trim() || null
+          });
+        }
+      }
+      return specs.filter(s => s.name);
+    });
+  },
+
+  star_rating: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) {
+      const val = (productData.aggregateRating as JsonObject).ratingValue;
+      if (val) return String(val);
+    }
+
+    return page.evaluate(() => {
+      const ratingEl = document.querySelector('.rating-value, [itemprop="ratingValue"], .stars');
+      if (ratingEl) return ratingEl.textContent?.trim() || ratingEl.getAttribute('content');
+      return null;
+    });
+  },
+
+  review_count: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.aggregateRating && typeof productData.aggregateRating === 'object' && !Array.isArray(productData.aggregateRating)) {
+      const val = (productData.aggregateRating as JsonObject).reviewCount;
+      if (val) return String(val);
+    }
+
+    return page.evaluate(() => {
+      const countEl = document.querySelector('.review-count, [itemprop="reviewCount"]');
+      if (countEl) return countEl.textContent?.trim() || countEl.getAttribute('content');
+      return null;
+    });
+  },
+
+  gtin: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.gtin13) return String(productData.gtin13);
+    if (productData?.gtin) return String(productData.gtin);
+
+    return page.evaluate(() => {
+      const el = document.querySelector('[itemprop="gtin13"], [itemprop="gtin"]');
+      return el ? (el.textContent?.trim() || el.getAttribute('content')) : null;
+    });
+  },
+
+  mpn: async (page) => {
+    const productData = await findJsonLdType(page, "Product");
+    if (productData?.mpn) return String(productData.mpn);
+
+    return page.evaluate(() => {
+      const el = document.querySelector('[itemprop="mpn"]');
+      return el ? (el.textContent?.trim() || el.getAttribute('content')) : null;
+    });
+  },
+
+  scraped_at: async () => {
+    return new Date().toISOString();
+  }
+};