From 77d946eab49f75e6f288212e049820ea7a97861d Mon Sep 17 00:00:00 2001 From: Hudson Riggs Date: Sun, 26 Oct 2025 09:37:17 -0400 Subject: [PATCH] broken price report --- package-lock.json | 290 +++++++++++++++++++++++++++++++++++++++++++ src/db/client.ts | 19 ++- src/routes/api.ts | 67 ++++++---- src/services/ebay.ts | 125 +++++++++++++++---- src/utils/sku.ts | 16 +++ 5 files changed, 466 insertions(+), 51 deletions(-) create mode 100644 src/utils/sku.ts diff --git a/package-lock.json b/package-lock.json index 3a2c4fd..2ee8542 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,6 +9,7 @@ "version": "1.0.0", "license": "ISC", "dependencies": { + "cheerio": "^1.0.0", "cors": "^2.8.5", "dotenv": "^17.2.3", "exceljs": "^4.4.0", @@ -20,6 +21,7 @@ "pdfkit": "^0.17.2" }, "devDependencies": { + "@types/cheerio": "^0.22.35", "@types/cors": "^2.8.19", "@types/express": "^5.0.4", "@types/express-fileupload": "^1.5.1", @@ -171,6 +173,16 @@ "@types/node": "*" } }, + "node_modules/@types/cheerio": { + "version": "0.22.35", + "resolved": "https://registry.npmjs.org/@types/cheerio/-/cheerio-0.22.35.tgz", + "integrity": "sha512-yD57BchKRvTV+JD53UZ6PD8KWY5g5rvvMLRnZR3EQBCZXiDT/HR+pKpMzFGlWNhFrXlo7VPZXtKvIEwZkAWOIA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/connect": { "version": "3.4.38", "resolved": "https://registry.npmjs.org/@types/connect/-/connect-3.4.38.tgz", @@ -579,6 +591,12 @@ "node": ">=18" } }, + "node_modules/boolbase": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/boolbase/-/boolbase-1.0.0.tgz", + "integrity": "sha512-JZOSA7Mo9sNGB8+UjSgzdLtokWAky1zbztM3WRLCbZ70/3cTANmQmOdR7y2g+J0e2WXywy1yS468tY+IruqEww==", + "license": "ISC" + }, "node_modules/brace-expansion": { "version": "1.1.12", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.12.tgz", @@ -722,6 +740,48 @@ "node": "*" } }, + "node_modules/cheerio": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/cheerio/-/cheerio-1.1.2.tgz", + "integrity": "sha512-IkxPpb5rS/d1IiLbHMgfPuS0FgiWTtFIm/Nj+2woXDLTZ7fOT2eqzgYbdMlLweqlHbsZjxEChoVK+7iph7jyQg==", + "license": "MIT", + "dependencies": { + "cheerio-select": "^2.1.0", + "dom-serializer": "^2.0.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.2", + "encoding-sniffer": "^0.2.1", + "htmlparser2": "^10.0.0", + "parse5": "^7.3.0", + "parse5-htmlparser2-tree-adapter": "^7.1.0", + "parse5-parser-stream": "^7.1.2", + "undici": "^7.12.0", + "whatwg-mimetype": "^4.0.0" + }, + "engines": { + "node": ">=20.18.1" + }, + "funding": { + "url": "https://github.com/cheeriojs/cheerio?sponsor=1" + } + }, + "node_modules/cheerio-select": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/cheerio-select/-/cheerio-select-2.1.0.tgz", + "integrity": "sha512-9v9kG0LvzrlcungtnJtpGNxY+fzECQKhK4EGJX2vByejiMX84MFNQw4UxPJl3bFbTMw+Dfs37XaIkCwTZfLh4g==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-select": "^5.1.0", + "css-what": "^6.1.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/chokidar": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/chokidar/-/chokidar-3.6.0.tgz", @@ -873,6 +933,34 @@ "integrity": "sha512-KALDyEYgpY+Rlob/iriUtjV6d5Eq+Y191A5g4UqLAi8CyGP9N1+FdVbkc1SxKc2r4YAYqG8JzO2KGL+AizD70Q==", "license": "MIT" }, + "node_modules/css-select": { + "version": "5.2.2", + "resolved": "https://registry.npmjs.org/css-select/-/css-select-5.2.2.tgz", + "integrity": "sha512-TizTzUddG/xYLA3NXodFM0fSbNizXjOKhqiQQwvhlspadZokn1KDy0NZFS0wuEubIYAV5/c1/lAr0TaaFXEXzw==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0", + "css-what": "^6.1.0", + "domhandler": "^5.0.2", + "domutils": "^3.0.1", + "nth-check": "^2.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, + "node_modules/css-what": { + "version": "6.2.2", + "resolved": "https://registry.npmjs.org/css-what/-/css-what-6.2.2.tgz", + "integrity": "sha512-u/O3vwbptzhMs3L1fQE82ZSLHQQfto5gyZzwteVIEyeaY5Fc7R4dapF/BvRoSYFeqfBk4m0V1Vafq5Pjv25wvA==", + "license": "BSD-2-Clause", + "engines": { + "node": ">= 6" + }, + "funding": { + "url": "https://github.com/sponsors/fb55" + } + }, "node_modules/dayjs": { "version": "1.11.18", "resolved": "https://registry.npmjs.org/dayjs/-/dayjs-1.11.18.tgz", @@ -930,6 +1018,61 @@ "node": ">=0.3.1" } }, + "node_modules/dom-serializer": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/dom-serializer/-/dom-serializer-2.0.0.tgz", + "integrity": "sha512-wIkAryiqt/nV5EQKqQpo3SToSOV9J0DnbJqwK7Wv/Trc92zIAYZ4FlMu+JPFW1DfGFt81ZTCGgDEabffXeLyJg==", + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.2", + "entities": "^4.2.0" + }, + "funding": { + "url": "https://github.com/cheeriojs/dom-serializer?sponsor=1" + } + }, + "node_modules/domelementtype": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/domelementtype/-/domelementtype-2.3.0.tgz", + "integrity": "sha512-OLETBj6w0OsagBwdXnPdN0cnMfF9opN69co+7ZrbfPGrdpPVNBUj02spi6B1N7wChLQiPn4CSH/zJvXw56gmHw==", + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "BSD-2-Clause" + }, + "node_modules/domhandler": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/domhandler/-/domhandler-5.0.3.tgz", + "integrity": "sha512-cgwlv/1iFQiFnU96XXgROh8xTeetsnJiDsTc7TYCLFd9+/WNkIqPTxiM/8pSd8VIrhXGTf1Ny1q1hquVqDJB5w==", + "license": "BSD-2-Clause", + "dependencies": { + "domelementtype": "^2.3.0" + }, + "engines": { + "node": ">= 4" + }, + "funding": { + "url": "https://github.com/fb55/domhandler?sponsor=1" + } + }, + "node_modules/domutils": { + "version": "3.2.2", + "resolved": "https://registry.npmjs.org/domutils/-/domutils-3.2.2.tgz", + "integrity": "sha512-6kZKyUajlDuqlHKVX1w7gyslj9MPIXzIFiz/rGu35uC1wMi+kMhQwGhl4lt9unC9Vb9INnY9Z3/ZA3+FhASLaw==", + "license": "BSD-2-Clause", + "dependencies": { + "dom-serializer": "^2.0.0", + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3" + }, + "funding": { + "url": "https://github.com/fb55/domutils?sponsor=1" + } + }, "node_modules/dotenv": { "version": "17.2.3", "resolved": "https://registry.npmjs.org/dotenv/-/dotenv-17.2.3.tgz", @@ -1010,6 +1153,19 @@ "node": ">= 0.8" } }, + "node_modules/encoding-sniffer": { + "version": "0.2.1", + "resolved": "https://registry.npmjs.org/encoding-sniffer/-/encoding-sniffer-0.2.1.tgz", + "integrity": "sha512-5gvq20T6vfpekVtqrYQsSCFZ1wEg5+wW0/QaZMWkFr6BqD3NfKs0rLCx4rrVlSWJeZb5NBJgVLswK/w2MWU+Gw==", + "license": "MIT", + "dependencies": { + "iconv-lite": "^0.6.3", + "whatwg-encoding": "^3.1.1" + }, + "funding": { + "url": "https://github.com/fb55/encoding-sniffer?sponsor=1" + } + }, "node_modules/end-of-stream": { "version": "1.4.5", "resolved": "https://registry.npmjs.org/end-of-stream/-/end-of-stream-1.4.5.tgz", @@ -1019,6 +1175,18 @@ "once": "^1.4.0" } }, + "node_modules/entities": { + "version": "4.5.0", + "resolved": "https://registry.npmjs.org/entities/-/entities-4.5.0.tgz", + "integrity": "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/es-define-property": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.1.tgz", @@ -1406,6 +1574,37 @@ "node": ">= 0.4" } }, + "node_modules/htmlparser2": { + "version": "10.0.0", + "resolved": "https://registry.npmjs.org/htmlparser2/-/htmlparser2-10.0.0.tgz", + "integrity": "sha512-TwAZM+zE5Tq3lrEHvOlvwgj1XLWQCtaaibSN11Q+gGBAS7Y1uZSWwXXRe4iF6OXnaq1riyQAPFOBtYc77Mxq0g==", + "funding": [ + "https://github.com/fb55/htmlparser2?sponsor=1", + { + "type": "github", + "url": "https://github.com/sponsors/fb55" + } + ], + "license": "MIT", + "dependencies": { + "domelementtype": "^2.3.0", + "domhandler": "^5.0.3", + "domutils": "^3.2.1", + "entities": "^6.0.0" + } + }, + "node_modules/htmlparser2/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/http-errors": { "version": "2.0.0", "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", @@ -2040,6 +2239,18 @@ "node": ">=0.10.0" } }, + "node_modules/nth-check": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/nth-check/-/nth-check-2.1.1.tgz", + "integrity": "sha512-lqjrjmaOoAnWfMmBPL+XNnynZh2+swxiX3WUE0s4yEHI6m+AwrK2UZOimIRl3X/4QctVqS8AiZjFqyOGrMXb/w==", + "license": "BSD-2-Clause", + "dependencies": { + "boolbase": "^1.0.0" + }, + "funding": { + "url": "https://github.com/fb55/nth-check?sponsor=1" + } + }, "node_modules/object-assign": { "version": "4.1.1", "resolved": "https://registry.npmjs.org/object-assign/-/object-assign-4.1.1.tgz", @@ -2118,6 +2329,55 @@ "integrity": "sha512-NUcwaKxUxWrZLpDG+z/xZaCgQITkA/Dv4V/T6bw7VON6l1Xz/VnrBqrYjZQ12TamKHzITTfOEIYUj48y2KXImA==", "license": "MIT" }, + "node_modules/parse5": { + "version": "7.3.0", + "resolved": "https://registry.npmjs.org/parse5/-/parse5-7.3.0.tgz", + "integrity": "sha512-IInvU7fabl34qmi9gY8XOVxhYyMyuH2xUNpb2q8/Y+7552KlejkRvqvD19nMoUW/uQGGbqNpA6Tufu5FL5BZgw==", + "license": "MIT", + "dependencies": { + "entities": "^6.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-htmlparser2-tree-adapter": { + "version": "7.1.0", + "resolved": "https://registry.npmjs.org/parse5-htmlparser2-tree-adapter/-/parse5-htmlparser2-tree-adapter-7.1.0.tgz", + "integrity": "sha512-ruw5xyKs6lrpo9x9rCZqZZnIUntICjQAd0Wsmp396Ul9lN/h+ifgVV1x1gZHi8euej6wTfpqX8j+BFQxF0NS/g==", + "license": "MIT", + "dependencies": { + "domhandler": "^5.0.3", + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5-parser-stream": { + "version": "7.1.2", + "resolved": "https://registry.npmjs.org/parse5-parser-stream/-/parse5-parser-stream-7.1.2.tgz", + "integrity": "sha512-JyeQc9iwFLn5TbvvqACIF/VXG6abODeB3Fwmv/TGdLk2LfbWkaySGY72at4+Ty7EkPZj854u4CrICqNk2qIbow==", + "license": "MIT", + "dependencies": { + "parse5": "^7.0.0" + }, + "funding": { + "url": "https://github.com/inikulin/parse5?sponsor=1" + } + }, + "node_modules/parse5/node_modules/entities": { + "version": "6.0.1", + "resolved": "https://registry.npmjs.org/entities/-/entities-6.0.1.tgz", + "integrity": "sha512-aN97NXWF6AWBTahfVOIrB/NShkzi5H7F9r1s9mD3cDj4Ko5f2qhhVoYMibXF7GlLveb/D2ioWay8lxI97Ven3g==", + "license": "BSD-2-Clause", + "engines": { + "node": ">=0.12" + }, + "funding": { + "url": "https://github.com/fb55/entities?sponsor=1" + } + }, "node_modules/parseurl": { "version": "1.3.3", "resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.3.tgz", @@ -2746,6 +3006,15 @@ "dev": true, "license": "MIT" }, + "node_modules/undici": { + "version": "7.16.0", + "resolved": "https://registry.npmjs.org/undici/-/undici-7.16.0.tgz", + "integrity": "sha512-QEg3HPMll0o3t2ourKwOeUAZ159Kn9mx5pnzHRQO8+Wixmh88YdZRiIwat0iNzNNXn0yoEtXJqFpyW7eM8BV7g==", + "license": "MIT", + "engines": { + "node": ">=20.18.1" + } + }, "node_modules/undici-types": { "version": "7.16.0", "resolved": "https://registry.npmjs.org/undici-types/-/undici-types-7.16.0.tgz", @@ -2861,6 +3130,27 @@ "node": ">= 0.8" } }, + "node_modules/whatwg-encoding": { + "version": "3.1.1", + "resolved": "https://registry.npmjs.org/whatwg-encoding/-/whatwg-encoding-3.1.1.tgz", + "integrity": "sha512-6qN4hJdMwfYBtE3YBTTHhoeuUrDBPZmbQaxWAqSALV/MeEnR5z1xd8UKud2RAkFoPkmB+hli1TZSnyi84xz1vQ==", + "license": "MIT", + "dependencies": { + "iconv-lite": "0.6.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/whatwg-mimetype": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/whatwg-mimetype/-/whatwg-mimetype-4.0.0.tgz", + "integrity": "sha512-QaKxh0eNIi2mE9p2vEdzfagOKHCcj1pJ56EEHGQOVxp8r9/iszLUUV7v89x9O1p/T+NlTM5W7jW6+cz4Fq1YVg==", + "license": "MIT", + "engines": { + "node": ">=18" + } + }, "node_modules/wrappy": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/wrappy/-/wrappy-1.0.2.tgz", diff --git a/src/db/client.ts b/src/db/client.ts index cdc3033..f0613f7 100644 --- a/src/db/client.ts +++ b/src/db/client.ts @@ -1,5 +1,6 @@ import { createPool, Pool } from 'mysql2/promise'; import { TrainItem } from '../services/openaiVision.js'; +import { normalizeSku } from '../utils/sku.js'; const pool: Pool = createPool({ host: process.env.db_ip, @@ -59,13 +60,14 @@ function getReady() { export const db = { async insertItem(item: Omit) { await getReady(); + const normalizedSku = normalizeSku((item as any).sku); const [result] = await pool.execute( `insert into items (manufacturer, model, sku, quantity, description, item_condition, has_box, image_path, image_data, image_mime) values (?,?,?,?,?,?,?,?,?,?)`, [ item.manufacturer, item.model, - item.sku || null, + normalizedSku, item.quantity, item.description, item.condition, @@ -142,15 +144,22 @@ export const db = { async listDistinctSkus() { await getReady(); const [rows] = await pool.query(`select distinct sku from items where sku is not null and sku <> ''`); - return (rows as any[]).map(r => r.sku as string); + const set = new Set(); + for (const r of rows as any[]) { + const s = normalizeSku(r.sku as string); + if (s) set.add(s); + } + return Array.from(set); }, async upsertSkuPrice(sku: string, price: number | null, currency: string = 'USD', source: string = 'ebay') { await getReady(); + const s = normalizeSku(sku); + if (!s) return; // nothing to upsert await pool.query( `insert into sku_prices (sku, price, currency, source, updated_at) values (?,?,?,?, current_timestamp) on duplicate key update price = values(price), currency = values(currency), source = values(source), updated_at = current_timestamp`, - [sku, price, currency, source] + [s, price, currency, source] ); }, async getSkuPriceMap() { @@ -158,7 +167,9 @@ export const db = { const [rows] = await pool.query(`select sku, price, currency, updated_at as updatedAt from sku_prices`); const map: Record = {}; for (const r of rows as any[]) { - map[r.sku] = { price: r.price !== null ? Number(r.price) : null, currency: r.currency, updatedAt: r.updatedAt }; + const key = normalizeSku(r.sku); + if (!key) continue; + map[key] = { price: r.price !== null ? Number(r.price) : null, currency: r.currency, updatedAt: r.updatedAt }; } return map; } diff --git a/src/routes/api.ts b/src/routes/api.ts index 28f8e98..3d89487 100644 --- a/src/routes/api.ts +++ b/src/routes/api.ts @@ -3,7 +3,8 @@ import path from 'path'; import fs from 'fs/promises'; import { analyzeImageToMetadata } from '../services/openaiVision.js'; import { db } from '../db/client.js'; -import { fetchMedianSoldPriceUSDForSku } from '../services/ebay.js'; +import { fetchMedianSoldPriceUSDForSku, debugFetchSoldPricesUSDForSku } from '../services/ebay.js'; +import { normalizeSku } from '../utils/sku.js'; import { generatePdfForItem } from '../services/pdf.js'; export const router = Router(); @@ -178,16 +179,18 @@ router.post('/prices/update', async (_req, res) => { try { const skus = await db.listDistinctSkus(); const out: Record = {}; - for (const sku of skus) { + for (const sku of skus) { try { - const price = await fetchMedianSoldPriceUSDForSku(sku); - await db.upsertSkuPrice(sku, price, 'USD', 'ebay'); - out[sku] = price; - // small delay to be gentle - await new Promise(r => setTimeout(r, 400)); + const norm = normalizeSku(sku); + if (!norm) { out[sku] = null; continue; } + const price = await fetchMedianSoldPriceUSDForSku(norm); + await db.upsertSkuPrice(norm, price, 'USD', 'ebay'); + out[norm] = price; + // small delay to be gentle (reduce block risk) + await new Promise(r => setTimeout(r, 900)); } catch (e) { - console.error('Failed to update price for', sku, e); - out[sku] = null; + console.error('Failed to update price for', sku, e); + out[sku] = null; } } res.json({ updated: out }); @@ -204,33 +207,49 @@ router.get('/price-report', async (_req, res) => { db.listItems(), db.getSkuPriceMap() ]); - const itemValues = items.map(it => { - const p = it.sku ? priceMap[it.sku]?.price ?? null : null; + const itemValues = items.map(it => { + const key = normalizeSku(it.sku); + const p = key ? priceMap[key]?.price ?? null : null; const unitPrice = p !== null ? Number(p) : null; const totalValue = unitPrice !== null ? Number((unitPrice * it.quantity).toFixed(2)) : 0; return { id: it.id, - sku: it.sku || null, + sku: key || null, description: it.description, quantity: it.quantity, unitPrice, totalValue }; }); + + // Debug: inspect parsed eBay prices for a given sku + router.get('/debug/ebay-prices', async (req, res) => { + try { + const skuParam = typeof req.query.sku === 'string' ? req.query.sku : ''; + const norm = normalizeSku(skuParam); + if (!norm) return res.status(400).json({ error: 'sku required' }); + const details = await debugFetchSoldPricesUSDForSku(norm); + res.json({ sku: norm, ...details }); + } catch (err: any) { + console.error(err); + res.status(500).json({ error: err.message || 'Failed to debug ebay prices' }); + } + }); // Build sku list with one description (first encountered) const skuListMap: Record = {}; - for (const it of items) { - if (!it.sku) continue; - if (!skuListMap[it.sku]) { - const p = priceMap[it.sku]; - skuListMap[it.sku] = { - sku: it.sku, - price: p ? (p.price !== null ? Number(p.price) : null) : null, - currency: p ? p.currency : 'USD', - updatedAt: p ? p.updatedAt : null, - description: it.description - }; - } + for (const it of items) { + const key = normalizeSku(it.sku); + if (!key) continue; + if (!skuListMap[key]) { + const p = priceMap[key]; + skuListMap[key] = { + sku: key, + price: p ? (p.price !== null ? Number(p.price) : null) : null, + currency: p ? p.currency : 'USD', + updatedAt: p ? p.updatedAt : null, + description: it.description + }; + } } const skuList = Object.values(skuListMap).sort((a, b) => (a.sku || '').localeCompare(b.sku || '')); res.json({ items: itemValues, skus: skuList }); diff --git a/src/services/ebay.ts b/src/services/ebay.ts index b149546..ce1003f 100644 --- a/src/services/ebay.ts +++ b/src/services/ebay.ts @@ -1,38 +1,117 @@ import * as cheerio from 'cheerio'; -export async function fetchMedianSoldPriceUSDForSku(sku: string): Promise { - const query = encodeURIComponent(`"${sku}"`); - const url = `https://www.ebay.com/sch/i.html?_nkw=${query}&LH_Sold=1&LH_Complete=1&rt=nc`; - const res = await fetch(url, { - headers: { - 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36' - } as any - } as any); - if (!res.ok) return null; - const html = await res.text(); +type ScrapeResult = { + prices: number[]; + blocked: boolean; + diagnostics?: { selectorHits: Record; sampleTexts: string[]; samplePrices: number[] }; +}; + +async function scrapeSoldPricesUSDPage(html: string, wantDiagnostics = false): Promise { const $ = cheerio.load(html); const prices: number[] = []; + const selectorHits: Record = {}; + const sampleTexts: string[] = []; + const samplePrices: number[] = []; + + const blocked = html.includes('To continue, please verify') || html.toLowerCase().includes('robot check'); + + const priceSelectors = [ + '.s-item__price', + '.s-item__detail--primary .s-item__price', + 'span[class*="s-item__price"]', + ]; + $('li.s-item').each((_i, el) => { - const priceText = $(el).find('.s-item__price').first().text().trim(); - if (!priceText) return; - // Accept only USD (has $) - if (!priceText.includes('$')) return; - // Remove ranges like "$10.00 to $20.00" - const single = priceText.split(' to ')[0]; + const $el = $(el); + let text: string | null = null; + for (const sel of priceSelectors) { + const t = $el.find(sel).first().text().trim(); + if (t) { + text = t; selectorHits[sel] = (selectorHits[sel] || 0) + 1; break; + } + } + if (!text) { + // regex fallback within this listing's HTML + const htmlFrag = $el.html() || ''; + const m = htmlFrag.match(/\$\s*[0-9]{1,3}(?:,[0-9]{3})*(?:\.[0-9]{2})?/); + if (m) text = m[0]; + } + if (!text) return; + if (wantDiagnostics && sampleTexts.length < 10) sampleTexts.push(text); + if (!text.includes('$')) return; + const single = text.split(' to ')[0]; const num = single.replace(/[^0-9.]/g, ''); if (!num) return; const value = Number(num); if (!Number.isFinite(value) || value <= 0) return; prices.push(value); + if (wantDiagnostics && samplePrices.length < 10) samplePrices.push(value); }); - if (prices.length === 0) return null; - prices.sort((a, b) => a - b); - const mid = Math.floor(prices.length / 2); - if (prices.length % 2 === 0) { - return Number(((prices[mid - 1] + prices[mid]) / 2).toFixed(2)); - } else { - return Number(prices[mid].toFixed(2)); + + return { prices, blocked, diagnostics: wantDiagnostics ? { selectorHits, sampleTexts, samplePrices } : undefined }; +} + +async function fetchSoldSearchHtml(query: string, page = 1): Promise<{ ok: boolean; html: string }> { + const url = `https://www.ebay.com/sch/i.html?_nkw=${encodeURIComponent(query)}&LH_Sold=1&LH_Complete=1&rt=nc&_ipg=200&_pgn=${page}`; + const res = await fetch(url, { + headers: { + 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36', + 'Accept-Language': 'en-US,en;q=0.9' + } as any + } as any); + const html = await res.text(); + return { ok: res.ok, html }; +} + +export async function fetchMedianSoldPriceUSDForSku(sku: string): Promise { + // Try quoted exact search first + const tryQueries = [ + `"${sku}"`, + sku // fallback without quotes + ]; + for (const q of tryQueries) { + let all: number[] = []; + for (let page = 1; page <= 2; page++) { + const { ok, html } = await fetchSoldSearchHtml(q, page); + if (!ok) continue; + const { prices, blocked } = await scrapeSoldPricesUSDPage(html); + if (blocked) return null; + all = all.concat(prices); + if (all.length === 0) { + // small delay before next page to be gentle + await new Promise(r => setTimeout(r, 600)); + } + } + if (all.length > 0) { + all.sort((a, b) => a - b); + const mid = Math.floor(all.length / 2); + return Number((all.length % 2 === 0 ? (all[mid - 1] + all[mid]) / 2 : all[mid]).toFixed(2)); + } } + return null; +} + +export async function debugFetchSoldPricesUSDForSku(sku: string) { + const sequences = [`"${sku}"`, sku]; + const attempts: any[] = []; + for (const q of sequences) { + let total = 0; + let blocked = false; + const diagnostics: any = { pages: [] }; + for (let page = 1; page <= 2; page++) { + const { ok, html } = await fetchSoldSearchHtml(q, page); + const diag = await scrapeSoldPricesUSDPage(html, true); + diagnostics.pages.push({ page, ok, count: diag.prices.length, selectorHits: diag.diagnostics?.selectorHits, sampleTexts: diag.diagnostics?.sampleTexts, samplePrices: diag.diagnostics?.samplePrices }); + total += diag.prices.length; + blocked = blocked || diag.blocked; + if (diag.prices.length === 0 && page === 1) { + await new Promise(r => setTimeout(r, 600)); + } + } + attempts.push({ query: q, totalCount: total, blocked, details: diagnostics }); + if (total > 0) break; + } + return { attempts }; } diff --git a/src/utils/sku.ts b/src/utils/sku.ts new file mode 100644 index 0000000..2e39917 --- /dev/null +++ b/src/utils/sku.ts @@ -0,0 +1,16 @@ +export function normalizeSku(raw: string | null | undefined): string | null { + if (!raw) return null; + const s = String(raw) + .trim() + .toUpperCase() + // normalize dash-like characters to hyphen + .replace(/[\u2010-\u2015\u2212\uFE58\uFE63\uFF0D]/g, '-') + // collapse multiple separators or spaces around hyphens + .replace(/\s*[-_\s]+\s*/g, '-') + // remove leading/trailing hyphens + .replace(/^-+/, '') + .replace(/-+$/, ''); + return s || null; +} + +